mach_commands.py (22728B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import argparse 7 import html 8 import json 9 import logging 10 import os 11 import re 12 import textwrap 13 import webbrowser 14 15 # Command files like this are listed in build/mach_initialize.py in alphabetical 16 # order, but we need to access commands earlier in the sorted order to grab 17 # their arguments. Force them to load now. 18 import mozbuild.artifact_commands # NOQA: F401 19 import mozbuild.build_commands # NOQA: F401 20 import mozhttpd 21 from mach.base import FailedCommandError, MachError 22 from mach.decorators import Command, CommandArgument, SubCommand 23 from mach.registrar import Registrar 24 from mozbuild.base import BuildEnvironmentNotFoundException 25 from mozbuild.mozconfig import MozconfigLoader 26 27 28 # Use a decorator to copy command arguments off of the named command. Instead 29 # of a decorator, this could be straight code that edits eg 30 # MachCommands.build_shell._mach_command.arguments, but that looked uglier. 31 def inherit_command_args(command, subcommand=None): 32 """Decorator for inheriting all command-line arguments from `mach build`. 33 34 This should come earlier in the source file than @Command or @SubCommand, 35 because it relies on that decorator having run first.""" 36 37 def inherited(func): 38 handler = Registrar.command_handlers.get(command) 39 if handler is not None and subcommand is not None: 40 handler = handler.subcommand_handlers.get(subcommand) 41 if handler is None: 42 raise MachError( 43 "{} command unknown or not yet loaded".format( 44 command if subcommand is None else command + " " + subcommand 45 ) 46 ) 47 func._mach_command.arguments.extend(handler.arguments) 48 return func 49 50 return inherited 51 52 53 def state_dir(): 54 return os.environ.get("MOZBUILD_STATE_PATH", os.path.expanduser("~/.mozbuild")) 55 56 57 def tools_dir(): 58 if os.environ.get("MOZ_FETCHES_DIR"): 59 # In automation, tools are provided by toolchain dependencies. 60 return os.path.join(os.environ["HOME"], os.environ["MOZ_FETCHES_DIR"]) 61 62 # In development, `mach hazard bootstrap` installs the tools separately 63 # to avoid colliding with the "main" compiler versions, which can 64 # change separately (and the precompiled sixgill and compiler version 65 # must match exactly). 66 return os.path.join(state_dir(), "hazard-tools") 67 68 69 def sixgill_dir(): 70 return os.path.join(tools_dir(), "sixgill") 71 72 73 def gcc_dir(): 74 return os.path.join(tools_dir(), "gcc") 75 76 77 def script_dir(command_context): 78 return os.path.join(command_context.topsrcdir, "js/src/devtools/rootAnalysis") 79 80 81 def get_work_dir(command_context, project, given): 82 if given is not None: 83 return given 84 return os.path.join(command_context.topsrcdir, "haz-" + project) 85 86 87 def get_objdir(command_context, kwargs): 88 project = kwargs["project"] 89 objdir = kwargs["haz_objdir"] 90 if objdir is None: 91 objdir = os.environ.get("HAZ_OBJDIR") 92 if objdir is None: 93 objdir = os.path.join(command_context.topsrcdir, "obj-analyzed-" + project) 94 return objdir 95 96 97 def ensure_dir_exists(dir): 98 os.makedirs(dir, exist_ok=True) 99 return dir 100 101 102 # Force the use of hazard-compatible installs of tools. 103 def setup_env_for_tools(env): 104 gccbin = os.path.join(gcc_dir(), "bin") 105 env["CC"] = os.path.join(gccbin, "gcc") 106 env["CXX"] = os.path.join(gccbin, "g++") 107 env["PATH"] = "{sixgill_dir}/usr/bin:{gccbin}:{PATH}".format( 108 sixgill_dir=sixgill_dir(), gccbin=gccbin, PATH=env["PATH"] 109 ) 110 111 112 def setup_env_for_shell(env, shell): 113 """Add JS shell directory to dynamic lib search path""" 114 for var in ("LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"): 115 env[var] = ":".join(p for p in (env.get(var), os.path.dirname(shell)) if p) 116 117 118 @Command( 119 "hazards", 120 category="build", 121 order="declaration", 122 description="Commands for running the static analysis for GC rooting hazards", 123 ) 124 def hazards(command_context): 125 """Commands related to performing the GC rooting hazard analysis""" 126 command_context._sub_mach(["help", "hazards"]) 127 return 1 128 129 130 @inherit_command_args("artifact", "toolchain") 131 @SubCommand( 132 "hazards", 133 "bootstrap", 134 description="Install prerequisites for the hazard analysis", 135 ) 136 def bootstrap(command_context, **kwargs): 137 orig_dir = os.getcwd() 138 os.chdir(ensure_dir_exists(tools_dir())) 139 try: 140 kwargs["from_build"] = ("linux64-gcc-10-sixgill", "linux64-gcc-10") 141 command_context._mach_context.commands.dispatch( 142 "artifact", command_context._mach_context, subcommand="toolchain", **kwargs 143 ) 144 finally: 145 os.chdir(orig_dir) 146 147 148 CLOBBER_CHOICES = {"objdir", "work", "shell", "all"} 149 150 151 @SubCommand("hazards", "clobber", description="Clean up hazard-related files") 152 @CommandArgument("--project", default="browser", help="Build the given project.") 153 @CommandArgument("--application", dest="project", help="Build the given project.") 154 @CommandArgument("--haz-objdir", default=None, help="Hazard analysis objdir.") 155 @CommandArgument( 156 "--work-dir", default=None, help="Directory for output and working files." 157 ) 158 @CommandArgument( 159 "what", 160 default=["objdir", "work"], 161 nargs="*", 162 help="Target to clobber, must be one of {{{}}} (default objdir and work).".format( 163 ", ".join(CLOBBER_CHOICES) 164 ), 165 ) 166 def clobber(command_context, what, **kwargs): 167 from mozbuild.controller.clobber import Clobberer 168 169 what = set(what) 170 if "all" in what: 171 what.update(CLOBBER_CHOICES) 172 invalid = what - CLOBBER_CHOICES 173 if invalid: 174 print( 175 "Unknown clobber target(s): {}. Choose from {{{}}}".format( 176 ", ".join(invalid), ", ".join(CLOBBER_CHOICES) 177 ) 178 ) 179 return 1 180 181 try: 182 substs = command_context.substs 183 except BuildEnvironmentNotFoundException: 184 substs = {} 185 186 if "objdir" in what: 187 objdir = get_objdir(command_context, kwargs) 188 print(f"removing {objdir}") 189 Clobberer(command_context.topsrcdir, objdir, substs).remove_objdir(full=True) 190 if "work" in what: 191 project = kwargs["project"] 192 work_dir = get_work_dir(command_context, project, kwargs["work_dir"]) 193 print(f"removing {work_dir}") 194 Clobberer(command_context.topsrcdir, work_dir, substs).remove_objdir(full=True) 195 if "shell" in what: 196 objdir = os.path.join(command_context.topsrcdir, "obj-haz-shell") 197 print(f"removing {objdir}") 198 Clobberer(command_context.topsrcdir, objdir, substs).remove_objdir(full=True) 199 200 201 @inherit_command_args("build") 202 @SubCommand( 203 "hazards", "build-shell", description="Build a shell for the hazard analysis" 204 ) 205 @CommandArgument( 206 "--mozconfig", 207 default=None, 208 metavar="FILENAME", 209 help="Build with the given mozconfig.", 210 ) 211 def build_shell(command_context, **kwargs): 212 """Build a JS shell to use to run the rooting hazard analysis.""" 213 # The JS shell requires some specific configuration settings to execute 214 # the hazard analysis code, and configuration is done via mozconfig. 215 # Subprocesses find MOZCONFIG in the environment, so we can't just 216 # modify the settings in this process's loaded version. Pass it through 217 # the environment. 218 219 default_mozconfig = "js/src/devtools/rootAnalysis/mozconfig.haz_shell" 220 mozconfig_path = ( 221 kwargs.pop("mozconfig", None) 222 or os.environ.get("MOZCONFIG") 223 or default_mozconfig 224 ) 225 mozconfig_path = os.path.join(command_context.topsrcdir, mozconfig_path) 226 loader = MozconfigLoader(command_context.topsrcdir) 227 mozconfig = loader.read_mozconfig(mozconfig_path) 228 229 # Validate the mozconfig settings in case the user overrode the default. 230 configure_args = mozconfig["configure_args"] 231 if "--enable-ctypes" not in configure_args: 232 raise FailedCommandError( 233 "ctypes required in hazard JS shell, mozconfig=" + mozconfig_path 234 ) 235 236 # Transmit the mozconfig location to build subprocesses. 237 os.environ["MOZCONFIG"] = mozconfig_path 238 239 setup_env_for_tools(os.environ) 240 241 # Set a default objdir for the shell, for developer builds. 242 os.environ.setdefault( 243 "MOZ_OBJDIR", os.path.join(command_context.topsrcdir, "obj-haz-shell") 244 ) 245 246 return command_context._mach_context.commands.dispatch( 247 "build", command_context._mach_context, **kwargs 248 ) 249 250 251 def read_json_file(filename): 252 with open(filename) as fh: 253 return json.load(fh) 254 255 256 def ensure_shell(command_context, objdir): 257 if objdir is None: 258 objdir = os.path.join(command_context.topsrcdir, "obj-haz-shell") 259 260 try: 261 binaries = read_json_file(os.path.join(objdir, "binaries.json")) 262 info = [b for b in binaries["programs"] if b["program"] == "js"][0] 263 return os.path.join(objdir, info["install_target"], "js") 264 except (OSError, KeyError): 265 raise FailedCommandError( 266 """\ 267 no shell found in %s -- must build the JS shell with `mach hazards build-shell` first""" 268 % objdir 269 ) 270 271 272 def validate_mozconfig(command_context, kwargs): 273 app = kwargs.pop("project") 274 default_mozconfig = "js/src/devtools/rootAnalysis/mozconfig.%s" % app 275 mozconfig_path = ( 276 kwargs.pop("mozconfig", None) 277 or os.environ.get("MOZCONFIG") 278 or default_mozconfig 279 ) 280 mozconfig_path = os.path.join(command_context.topsrcdir, mozconfig_path) 281 282 loader = MozconfigLoader(command_context.topsrcdir) 283 mozconfig = loader.read_mozconfig(mozconfig_path) 284 configure_args = mozconfig["configure_args"] 285 286 # Require an explicit --enable-project/application=APP (even if you just 287 # want to build the default browser project.) 288 if ( 289 "--enable-project=%s" % app not in configure_args 290 and "--enable-application=%s" % app not in configure_args 291 ): 292 raise FailedCommandError( 293 textwrap.dedent( 294 f"""\ 295 mozconfig {mozconfig_path} builds wrong project. 296 unset MOZCONFIG to use the default {default_mozconfig}\ 297 """ 298 ) 299 ) 300 301 if not any("--with-compiler-wrapper" in a for a in configure_args): 302 raise FailedCommandError( 303 "mozconfig must wrap compiles with --with-compiler-wrapper" 304 ) 305 306 return mozconfig_path 307 308 309 @inherit_command_args("build") 310 @SubCommand( 311 "hazards", 312 "gather", 313 description="Gather analysis data by compiling the given project", 314 ) 315 @CommandArgument("--project", default="browser", help="Build the given project.") 316 @CommandArgument("--application", dest="project", help="Build the given project.") 317 @CommandArgument( 318 "--haz-objdir", default=None, help="Write object files to this directory." 319 ) 320 @CommandArgument( 321 "--work-dir", default=None, help="Directory for output and working files." 322 ) 323 def gather_hazard_data(command_context, **kwargs): 324 """Gather analysis information by compiling the tree""" 325 project = kwargs["project"] 326 objdir = get_objdir(command_context, kwargs) 327 328 work_dir = get_work_dir(command_context, project, kwargs["work_dir"]) 329 ensure_dir_exists(work_dir) 330 with open(os.path.join(work_dir, "defaults.py"), "w") as fh: 331 data = textwrap.dedent( 332 """\ 333 analysis_scriptdir = "{script_dir}" 334 objdir = "{objdir}" 335 source = "{srcdir}" 336 sixgill = "{sixgill_dir}/usr/libexec/sixgill" 337 sixgill_bin = "{sixgill_dir}/usr/bin" 338 """ 339 ).format( 340 script_dir=script_dir(command_context), 341 objdir=objdir, 342 srcdir=command_context.topsrcdir, 343 sixgill_dir=sixgill_dir(), 344 gcc_dir=gcc_dir(), 345 ) 346 fh.write(data) 347 348 buildscript = " ".join([ 349 command_context.topsrcdir + "/mach hazards compile", 350 *kwargs.get("what", []), 351 "--job-size=3.0", # Conservatively estimate 3GB/process 352 "--project=" + project, 353 "--haz-objdir=" + objdir, 354 ]) 355 args = [ 356 os.path.join(script_dir(command_context), "run_complete"), 357 "--foreground", 358 "--no-logs", 359 "--build-root=" + objdir, 360 "--wrap-dir=" + sixgill_dir() + "/usr/libexec/sixgill/scripts/wrap_gcc", 361 "--work-dir=work", 362 "-b", 363 sixgill_dir() + "/usr/bin", 364 "--buildcommand=" + buildscript, 365 ".", 366 ] 367 368 return command_context.run_process(args=args, cwd=work_dir, pass_thru=True) 369 370 371 @inherit_command_args("build") 372 @SubCommand("hazards", "compile", description=argparse.SUPPRESS) 373 @CommandArgument( 374 "--mozconfig", 375 default=None, 376 metavar="FILENAME", 377 help="Build with the given mozconfig.", 378 ) 379 @CommandArgument("--project", default="browser", help="Build the given project.") 380 @CommandArgument("--application", dest="project", help="Build the given project.") 381 @CommandArgument( 382 "--haz-objdir", 383 default=os.environ.get("HAZ_OBJDIR"), 384 help="Write object files to this directory.", 385 ) 386 def inner_compile(command_context, **kwargs): 387 """Build a source tree and gather analysis information while running 388 under the influence of the analysis collection server.""" 389 390 env = os.environ 391 392 # Check whether we are running underneath the manager (and therefore 393 # have a server to talk to). 394 if "XGILL_CONFIG" not in env: 395 raise FailedCommandError( 396 "no sixgill manager detected. `mach hazards compile` " 397 + "should only be run from `mach hazards gather`" 398 ) 399 400 mozconfig_path = validate_mozconfig(command_context, kwargs) 401 402 # Communicate mozconfig to build subprocesses. 403 env["MOZCONFIG"] = os.path.join(command_context.topsrcdir, mozconfig_path) 404 405 # hazard mozconfigs need to find binaries in .mozbuild 406 env["MOZBUILD_STATE_PATH"] = state_dir() 407 408 # Suppress the gathering of sources, to save disk space and memory. 409 env["XGILL_NO_SOURCE"] = "1" 410 411 setup_env_for_tools(env) 412 413 if "haz_objdir" in kwargs: 414 env["MOZ_OBJDIR"] = kwargs.pop("haz_objdir") 415 416 return command_context._mach_context.commands.dispatch( 417 "build", command_context._mach_context, **kwargs 418 ) 419 420 421 @SubCommand( 422 "hazards", "analyze", description="Analyzed gathered data for rooting hazards" 423 ) 424 @CommandArgument( 425 "--project", 426 default="browser", 427 help="Analyze the output for the given project.", 428 ) 429 @CommandArgument("--application", dest="project", help="Build the given project.") 430 @CommandArgument( 431 "--shell-objdir", 432 default=None, 433 help="objdir containing the optimized JS shell for running the analysis.", 434 ) 435 @CommandArgument( 436 "--work-dir", default=None, help="Directory for output and working files." 437 ) 438 @CommandArgument( 439 "--jobs", "-j", default=None, type=int, help="Number of parallel analyzers." 440 ) 441 @CommandArgument( 442 "--verbose", 443 "-v", 444 default=False, 445 action="store_true", 446 help="Display executed commands.", 447 ) 448 @CommandArgument( 449 "--from-stage", 450 default=None, 451 help="Stage to begin running at ('list' to see all).", 452 ) 453 @CommandArgument( 454 "extra", 455 nargs=argparse.REMAINDER, 456 default=(), 457 help="Remaining non-optional arguments to analyze.py script", 458 ) 459 def analyze( 460 command_context, 461 project, 462 shell_objdir, 463 work_dir, 464 jobs, 465 verbose, 466 from_stage, 467 extra, 468 ): 469 """Analyzed gathered data for rooting hazards""" 470 471 shell = ensure_shell(command_context, shell_objdir) 472 args = [ 473 os.path.join(script_dir(command_context), "analyze.py"), 474 "--js", 475 shell, 476 *extra, 477 ] 478 479 if from_stage is None: 480 pass 481 elif from_stage == "list": 482 args.append("--list") 483 else: 484 args.extend(["--first", from_stage]) 485 486 if jobs is not None: 487 args.extend(["-j", jobs]) 488 489 if verbose: 490 args.append("-v") 491 492 setup_env_for_tools(os.environ) 493 setup_env_for_shell(os.environ, shell) 494 495 work_dir = get_work_dir(command_context, project, work_dir) 496 return command_context.run_process(args=args, cwd=work_dir, pass_thru=True) 497 498 499 @SubCommand( 500 "hazards", 501 "self-test", 502 description="Run a self-test to verify hazards are detected", 503 ) 504 @CommandArgument( 505 "--shell-objdir", 506 default=None, 507 help="objdir containing the optimized JS shell for running the analysis.", 508 ) 509 @CommandArgument( 510 "extra", 511 nargs=argparse.REMAINDER, 512 help="Remaining non-optional arguments to pass to run-test.py", 513 ) 514 def self_test(command_context, shell_objdir, extra): 515 """Analyzed gathered data for rooting hazards""" 516 shell = ensure_shell(command_context, shell_objdir) 517 args = [ 518 os.path.join(script_dir(command_context), "run-test.py"), 519 "-v", 520 "--js", 521 shell, 522 "--sixgill", 523 os.path.join(tools_dir(), "sixgill"), 524 "--gccdir", 525 gcc_dir(), 526 ] 527 args.extend(extra) 528 529 setup_env_for_tools(os.environ) 530 setup_env_for_shell(os.environ, shell) 531 532 return command_context.run_process(args=args, pass_thru=True) 533 534 535 def annotated_source(filename, query): 536 """The index page has URLs of the format <http://.../path/to/source.cpp?L=m-n#m>. 537 The `#m` part will be stripped off and used by the browser to jump to the correct line. 538 The `?L=m-n` or `?L=m` parameter will be processed here on the server to highlight 539 the given line range.""" 540 linequery = query.replace("L=", "") 541 if "-" in linequery: 542 line0, line1 = linequery.split("-", 1) 543 else: 544 line0, line1 = linequery or "0", linequery or "0" 545 line0 = int(line0) 546 line1 = int(line1) 547 548 fh = open(filename) 549 550 out = "<pre>" 551 for lineno, line in enumerate(fh, 1): 552 processed = f"{lineno} <span id='{lineno}'" 553 if line0 <= lineno <= line1: 554 processed += " style='background: yellow'" 555 processed += ">" + html.escape(line.rstrip()) + "</span>\n" 556 out += processed 557 558 return out 559 560 561 @SubCommand( 562 "hazards", "view", description="Display a web page describing any hazards found" 563 ) 564 @CommandArgument( 565 "--project", 566 default="browser", 567 help="Analyze the output for the given project.", 568 ) 569 @CommandArgument("--application", dest="project", help="Build the given project.") 570 @CommandArgument( 571 "--haz-objdir", default=None, help="Write object files to this directory." 572 ) 573 @CommandArgument( 574 "--work-dir", default=None, help="Directory for output and working files." 575 ) 576 @CommandArgument("--port", default=6006, help="Port of the web server") 577 @CommandArgument( 578 "--serve-only", 579 default=False, 580 action="store_true", 581 help="Serve only, do not navigate to page", 582 ) 583 def view_hazards(command_context, project, haz_objdir, work_dir, port, serve_only): 584 work_dir = get_work_dir(command_context, project, work_dir) 585 haztop = os.path.basename(work_dir) 586 if haz_objdir is None: 587 haz_objdir = os.environ.get("HAZ_OBJDIR") 588 if haz_objdir is None: 589 haz_objdir = os.path.join(command_context.topsrcdir, "obj-analyzed-" + project) 590 591 httpd = None 592 593 def serve_source_file(request, path): 594 info = {"req": path} 595 596 def log(fmt, level=logging.INFO): 597 return command_context.log(level, "view-hazards", info, fmt) 598 599 if path in ("", f"{haztop}"): 600 info["dest"] = f"/{haztop}/hazards.html" 601 info["code"] = 301 602 log("serve '{req}' -> {code} {dest}") 603 return (info["code"], {"Location": info["dest"]}, "") 604 605 # Allow files to be served from the source directory or the objdir. 606 roots = (command_context.topsrcdir, haz_objdir) 607 608 try: 609 # Validate the path. Some source files have weird characters in their paths (eg "+"), but they 610 # all start with an alphanumeric or underscore. 611 command_context.log( 612 logging.DEBUG, "view-hazards", {"path": path}, "Raw path: {path}" 613 ) 614 path_component = r"\w[\w\-\.\+]*" 615 if not re.match(f"({path_component}/)*{path_component}$", path): 616 raise ValueError("invalid path") 617 618 # Resolve the path to under one of the roots, and 619 # ensure that the actual file really is underneath a root directory. 620 for rootdir in roots: 621 fullpath = os.path.join(rootdir, path) 622 info["path"] = fullpath 623 fullpath = os.path.realpath(fullpath) 624 if os.path.isfile(fullpath): 625 # symlinks between roots are ok, but not symlinks outside of the roots. 626 tops = [ 627 d 628 for d in roots 629 if fullpath.startswith(os.path.realpath(d) + "/") 630 ] 631 if len(tops) > 0: 632 break # Found a file underneath a root. 633 else: 634 raise OSError("not found") 635 636 html = annotated_source(fullpath, request.query) 637 log("serve '{req}' -> 200 {path}") 638 return ( 639 200, 640 {"Content-type": "text/html", "Content-length": len(html)}, 641 html, 642 ) 643 except (OSError, ValueError): 644 log("serve '{req}' -> 404 {path}", logging.ERROR) 645 return ( 646 404, 647 {"Content-type": "text/plain"}, 648 "We don't have that around here. Don't be asking for it.", 649 ) 650 651 httpd = mozhttpd.MozHttpd( 652 port=port, 653 docroot=None, 654 path_mappings={"/" + haztop: work_dir}, 655 urlhandlers=[ 656 # Treat everything not starting with /haz-browser/ (or /haz-js/) 657 # as a source file to be processed. Everything else is served 658 # as a plain file. 659 { 660 "method": "GET", 661 "path": "/(?!haz-" + project + "/)(.*)", 662 "function": serve_source_file, 663 }, 664 ], 665 log_requests=True, 666 ) 667 668 # The mozhttpd request handler class eats log messages. 669 httpd.handler_class.log_message = lambda self, format, *args: command_context.log( 670 logging.INFO, "view-hazards", {}, format % args 671 ) 672 673 print("Serving at %s:%s" % (httpd.host, httpd.port)) 674 675 httpd.start(block=False) 676 url = httpd.get_url(f"/{haztop}/hazards.html") 677 display_url = True 678 if not serve_only: 679 try: 680 webbrowser.get().open_new_tab(url) 681 display_url = False 682 except Exception: 683 pass 684 if display_url: 685 print("Please open %s in a browser." % url) 686 687 print("Hit CTRL+c to stop server.") 688 httpd.server.join()