mach_commands.py (24105B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import argparse 6 import json 7 import os 8 import platform 9 import re 10 import shutil 11 import subprocess 12 import sys 13 import tempfile 14 from collections import OrderedDict 15 16 import mozlog 17 import mozprofile 18 from mach.decorators import Command, CommandArgument, SubCommand 19 from mozbuild import nodeutil 20 from mozbuild.base import BinaryNotFoundException, MozbuildObject 21 22 EX_CONFIG = 78 23 EX_SOFTWARE = 70 24 EX_USAGE = 64 25 26 27 def setup(): 28 # add node and npm from mozbuild to front of system path 29 npm, _ = nodeutil.find_npm_executable() 30 if not npm: 31 exit(EX_CONFIG, "could not find npm executable") 32 path = os.path.abspath(os.path.join(npm, os.pardir)) 33 os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"]) 34 35 36 def remotedir(command_context): 37 return os.path.join(command_context.topsrcdir, "remote") 38 39 40 @Command("remote", category="misc", description="Remote protocol related operations.") 41 def remote(command_context): 42 """The remote subcommands all relate to the remote protocol.""" 43 command_context._sub_mach(["help", "remote"]) 44 return 1 45 46 47 @SubCommand( 48 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client." 49 ) 50 @CommandArgument( 51 "--repository", 52 metavar="REPO", 53 default="https://github.com/puppeteer/puppeteer.git", 54 help="The (possibly local) repository to clone from.", 55 ) 56 @CommandArgument( 57 "--commitish", 58 metavar="COMMITISH", 59 required=True, 60 help="The commit or tag object name to check out.", 61 ) 62 @CommandArgument( 63 "--no-install", 64 dest="install", 65 action="store_false", 66 default=True, 67 help="Do not install the just-pulled Puppeteer package,", 68 ) 69 def vendor_puppeteer(command_context, repository, commitish, install): 70 puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer") 71 72 # Preserve our custom mocha reporter 73 shutil.move( 74 os.path.join(puppeteer_dir, "json-mocha-reporter.js"), 75 os.path.join(remotedir(command_context), "json-mocha-reporter.js"), 76 ) 77 78 print("Removing folders for current Puppeteer version…") 79 shutil.rmtree(puppeteer_dir, ignore_errors=True) 80 os.makedirs(puppeteer_dir) 81 82 with TemporaryDirectory() as tmpdir: 83 print(f'Fetching commitish "{commitish}" from {repository}…') 84 git("clone", "--depth", "1", "--branch", commitish, repository, tmpdir) 85 git( 86 "checkout-index", 87 "-a", 88 "-f", 89 "--prefix", 90 f"{puppeteer_dir}/", 91 worktree=tmpdir, 92 ) 93 94 # remove files which may interfere with git checkout of central 95 try: 96 os.remove(os.path.join(puppeteer_dir, ".gitattributes")) 97 os.remove(os.path.join(puppeteer_dir, ".gitignore")) 98 except OSError: 99 pass 100 101 unwanted_dirs = ["experimental", "docs"] 102 103 for dir in unwanted_dirs: 104 dir_path = os.path.join(puppeteer_dir, dir) 105 if os.path.isdir(dir_path): 106 shutil.rmtree(dir_path) 107 108 shutil.move( 109 os.path.join(remotedir(command_context), "json-mocha-reporter.js"), 110 puppeteer_dir, 111 ) 112 113 import yaml 114 115 annotation = { 116 "schema": 1, 117 "bugzilla": { 118 "product": "Remote Protocol", 119 "component": "Agent", 120 }, 121 "origin": { 122 "name": "puppeteer", 123 "description": "Headless Chrome Node API", 124 "url": repository, 125 "license": "Apache-2.0", 126 "release": commitish, 127 }, 128 } 129 with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh: 130 yaml.safe_dump( 131 annotation, 132 fh, 133 default_flow_style=False, 134 encoding="utf-8", 135 allow_unicode=True, 136 ) 137 138 if install: 139 env = { 140 "CI": "1", # Force the quiet logger of wireit 141 "HUSKY": "0", # Disable any hook checks 142 "PUPPETEER_SKIP_DOWNLOAD": "1", # Don't download any build 143 } 144 145 print("Cleaning up and installing new version of Puppeteer…") 146 run_npm( 147 "run", 148 "clean", 149 cwd=puppeteer_dir, 150 env=env, 151 exit_on_fail=False, 152 ) 153 154 # Always use the `ci` command to not get updated sub-dependencies installed. 155 run_npm( 156 "ci", 157 cwd=os.path.join(command_context.topsrcdir, puppeteer_dir), 158 env=env, 159 ) 160 161 162 def git(*args, **kwargs): 163 cmd = ("git",) 164 if kwargs.get("worktree"): 165 cmd += ("-C", kwargs["worktree"]) 166 cmd += args 167 168 pipe = kwargs.get("pipe") 169 git_p = subprocess.Popen( 170 cmd, 171 env={"GIT_CONFIG_NOSYSTEM": "1"}, 172 stdout=subprocess.PIPE, 173 stderr=subprocess.PIPE, 174 ) 175 pipe_p = None 176 if pipe: 177 pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE) 178 179 if pipe: 180 _, pipe_err = pipe_p.communicate() 181 out, git_err = git_p.communicate() 182 183 # use error from first program that failed 184 if git_p.returncode > 0: 185 exit(EX_SOFTWARE, git_err) 186 if pipe and pipe_p.returncode > 0: 187 exit(EX_SOFTWARE, pipe_err) 188 189 return out 190 191 192 def run_npm(*args, **kwargs): 193 from mozprocess import run_and_wait 194 195 def output_timeout_handler(proc): 196 # In some cases, we wait longer for a mocha timeout 197 print( 198 "Timed out after {} seconds of no output".format(kwargs["output_timeout"]) 199 ) 200 201 env = os.environ.copy() 202 npm, _ = nodeutil.find_npm_executable() 203 if kwargs.get("env"): 204 env.update(kwargs["env"]) 205 206 proc_kwargs = {"output_timeout_handler": output_timeout_handler} 207 for kw in ["output_line_handler", "output_timeout"]: 208 if kw in kwargs: 209 proc_kwargs[kw] = kwargs[kw] 210 211 cmd = [npm] 212 cmd.extend(list(args)) 213 214 p = run_and_wait( 215 args=cmd, 216 cwd=kwargs.get("cwd"), 217 env=env, 218 text=True, 219 **proc_kwargs, 220 ) 221 post_wait_proc(p, cmd=npm, exit_on_fail=kwargs.get("exit_on_fail", True)) 222 223 return p.returncode 224 225 226 def post_wait_proc(p, cmd=None, exit_on_fail=True): 227 if p.poll() is None: 228 p.kill() 229 if exit_on_fail and p.returncode > 0: 230 msg = ( 231 "%s: exit code %s" % (cmd, p.returncode) 232 if cmd 233 else "exit code %s" % p.returncode 234 ) 235 exit(p.returncode, msg) 236 237 238 class MochaOutputHandler: 239 def __init__(self, logger, expected): 240 self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook') 241 242 self.logger = logger 243 self.proc = None 244 self.test_results = OrderedDict() 245 self.expected = expected 246 self.unexpected_skips = set() 247 248 self.has_unexpected = False 249 self.logger.suite_start([], name="puppeteer-tests") 250 self.status_map = { 251 "CRASHED": "CRASH", 252 "OK": "PASS", 253 "TERMINATED": "CRASH", 254 "pass": "PASS", 255 "fail": "FAIL", 256 "pending": "SKIP", 257 } 258 259 @property 260 def pid(self): 261 return self.proc and self.proc.pid 262 263 def __call__(self, proc, line): 264 self.proc = proc 265 line = line.rstrip("\r\n") 266 event = None 267 try: 268 if line.startswith("[") and line.endswith("]"): 269 event = json.loads(line) 270 self.process_event(event) 271 except ValueError: 272 pass 273 finally: 274 self.logger.process_output(self.pid, line, command="npm") 275 276 def testExpectation(self, testIdPattern, expected_name): 277 if testIdPattern.find("*") == -1: 278 return expected_name == testIdPattern 279 else: 280 return re.compile(re.escape(testIdPattern).replace(r"\*", ".*")).search( 281 expected_name 282 ) 283 284 def process_event(self, event): 285 if isinstance(event, list) and len(event) > 1: 286 status = self.status_map.get(event[0]) 287 test_start = event[0] == "test-start" 288 if not status and not test_start: 289 return 290 test_info = event[1] 291 test_full_title = test_info.get("fullTitle", "") 292 test_name = test_full_title 293 test_path = test_info.get("file", "") 294 test_file_name = os.path.basename(test_path).replace(".js", "") 295 test_err = test_info.get("err") 296 if status == "FAIL" and test_err: 297 if "timeout" in test_err.lower(): 298 status = "TIMEOUT" 299 if test_name and test_path: 300 test_name = f"{test_name} ({os.path.basename(test_path)})" 301 # mocha hook failures are not tracked in metadata 302 if status != "PASS" and self.hook_re.search(test_name): 303 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,)) 304 return 305 if test_start: 306 self.logger.test_start(test_name) 307 return 308 expected_name = f"[{test_file_name}] {test_full_title}" 309 expected_item = next( 310 ( 311 expectation 312 for expectation in reversed(list(self.expected)) 313 if self.testExpectation(expectation["testIdPattern"], expected_name) 314 ), 315 None, 316 ) 317 if expected_item is None: 318 expected = ["PASS"] 319 else: 320 expected = expected_item["expectations"] 321 # mozlog doesn't really allow unexpected skip, 322 # so if a test is disabled just expect that and note the unexpected skip 323 # Also, mocha doesn't log test-start for skipped tests 324 if status == "SKIP": 325 self.logger.test_start(test_name) 326 if self.expected and status not in expected: 327 self.unexpected_skips.add(test_name) 328 expected = ["SKIP"] 329 known_intermittent = expected[1:] 330 expected_status = expected[0] 331 332 # check if we've seen a result for this test before this log line 333 result_recorded = self.test_results.get(test_name) 334 if result_recorded: 335 self.logger.warning( 336 f"Received a second status for {test_name}: " 337 f"first {result_recorded}, now {status}" 338 ) 339 # mocha intermittently logs an additional test result after the 340 # test has already timed out. Avoid recording this second status. 341 if result_recorded != "TIMEOUT": 342 self.test_results[test_name] = status 343 if status not in expected: 344 self.has_unexpected = True 345 self.logger.test_end( 346 test_name, 347 status=status, 348 expected=expected_status, 349 known_intermittent=known_intermittent, 350 ) 351 352 def after_end(self): 353 if self.unexpected_skips: 354 self.has_unexpected = True 355 for test_name in self.unexpected_skips: 356 self.logger.error( 357 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,) 358 ) 359 self.logger.suite_end() 360 361 362 # tempfile.TemporaryDirectory missing from Python 2.7 363 class TemporaryDirectory: 364 def __init__(self): 365 self.path = tempfile.mkdtemp() 366 self._closed = False 367 368 def __repr__(self): 369 return f"<{self.__class__.__name__} {self.path!r}>" 370 371 def __enter__(self): 372 return self.path 373 374 def __exit__(self, exc, value, tb): 375 self.clean() 376 377 def __del__(self): 378 self.clean() 379 380 def clean(self): 381 if self.path and not self._closed: 382 shutil.rmtree(self.path) 383 self._closed = True 384 385 386 class PuppeteerRunner(MozbuildObject): 387 def __init__(self, *args, **kwargs): 388 super().__init__(*args, **kwargs) 389 390 self.remotedir = os.path.join(self.topsrcdir, "remote") 391 self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer") 392 393 def run_test(self, logger, *tests, **params): 394 """ 395 Runs Puppeteer unit tests with npm. 396 397 Possible optional test parameters: 398 399 `binary`: 400 Path for the browser binary to use. Defaults to the local 401 build. 402 `headless`: 403 Boolean to indicate whether to activate Firefox' headless mode. 404 `extra_prefs`: 405 Dictionary of extra preferences to write to the profile, 406 before invoking npm. Overrides default preferences. 407 `enable_webrender`: 408 Boolean to indicate whether to enable WebRender compositor in Gecko. 409 """ 410 setup() 411 412 binary = params.get("binary") 413 headless = params.get("headless", False) 414 product = params.get("product", "firefox") 415 this_chunk = params.get("this_chunk", "1") 416 total_chunks = params.get("total_chunks", "1") 417 418 extra_options = {} 419 for k, v in params.get("extra_launcher_options", {}).items(): 420 extra_options[k] = json.loads(v) 421 422 # Override upstream defaults: no retries, shorter timeout 423 mocha_options = [ 424 "--reporter", 425 "./json-mocha-reporter.js", 426 "--retries", 427 "0", 428 "--fullTrace", 429 "--timeout", 430 "20000", 431 "--no-parallel", 432 "--no-coverage", 433 ] 434 435 env = { 436 # Checked by Puppeteer's custom mocha config 437 "CI": "1", 438 # Print browser process output 439 "DUMPIO": "1", 440 # Run in headless mode if trueish, otherwise use headful 441 "HEADLESS": str(headless), 442 # Causes some tests to be skipped due to assumptions about install 443 "PUPPETEER_ALT_INSTALL": "1", 444 } 445 446 if product == "firefox": 447 env["BINARY"] = binary or self.get_binary_path() 448 env["PUPPETEER_PRODUCT"] = "firefox" 449 env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False) 450 else: 451 if binary: 452 env["BINARY"] = binary 453 env["PUPPETEER_CACHE_DIR"] = os.path.join( 454 self.topobjdir, 455 "_tests", 456 "remote", 457 "test", 458 "puppeteer", 459 ".cache", 460 ) 461 462 if product == "chrome": 463 if not headless: 464 raise Exception( 465 "Chrome doesn't support headful mode with the WebDriver BiDi protocol" 466 ) 467 test_command = "chrome-bidi" 468 elif product == "firefox": 469 if headless: 470 test_command = "firefox-headless" 471 else: 472 test_command = "firefox-headful" 473 else: 474 test_command = product 475 476 command = [ 477 "run", 478 "test", 479 "--", 480 "--shard", 481 f"{this_chunk}-{total_chunks}", 482 "--test-suite", 483 test_command, 484 ] + mocha_options 485 486 prefs = {} 487 for k, v in params.get("extra_prefs", {}).items(): 488 print(f"Using extra preference: {k}={v}") 489 prefs[k] = mozprofile.Preferences.cast(v) 490 491 if prefs: 492 extra_options["extraPrefsFirefox"] = prefs 493 494 if extra_options: 495 env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options) 496 497 expected_path = os.path.join( 498 os.path.dirname(__file__), 499 "test", 500 "puppeteer", 501 "test", 502 "TestExpectations.json", 503 ) 504 if os.path.exists(expected_path): 505 with open(expected_path) as f: 506 expected_data = json.load(f) 507 else: 508 expected_data = [] 509 510 expected_platform = platform.uname().system.lower() 511 if expected_platform == "windows": 512 expected_platform = "win32" 513 514 # Filter expectation data for the selected browser, 515 # headless or headful mode, the operating system, 516 # run in BiDi mode or not. 517 expectations = [ 518 expectation 519 for expectation in expected_data 520 if is_relevant_expectation( 521 expectation, product, env["HEADLESS"], expected_platform 522 ) 523 ] 524 525 output_handler = MochaOutputHandler(logger, expectations) 526 run_npm( 527 *command, 528 cwd=self.puppeteer_dir, 529 env=env, 530 output_line_handler=output_handler, 531 # Puppeteer unit tests don't always clean-up child processes in case of 532 # failure, so use an output_timeout as a fallback 533 output_timeout=60, 534 exit_on_fail=True, 535 ) 536 537 output_handler.after_end() 538 539 if output_handler.has_unexpected: 540 logger.error("Got unexpected results") 541 exit(1) 542 543 544 def create_parser_puppeteer(): 545 p = argparse.ArgumentParser() 546 p.add_argument( 547 "--product", type=str, default="firefox", choices=["chrome", "firefox"] 548 ) 549 p.add_argument( 550 "--binary", 551 type=str, 552 help="Path to browser binary. Defaults to local Firefox build.", 553 ) 554 p.add_argument( 555 "--ci", 556 action="store_true", 557 help="Flag that indicates that tests run in a CI environment.", 558 ) 559 p.add_argument( 560 "--disable-fission", 561 action="store_true", 562 default=False, 563 dest="disable_fission", 564 help="Disable Fission (site isolation) in Gecko.", 565 ) 566 p.add_argument( 567 "--enable-webrender", 568 action="store_true", 569 help="Enable the WebRender compositor in Gecko.", 570 ) 571 p.add_argument( 572 "-z", "--headless", action="store_true", help="Run browser in headless mode." 573 ) 574 p.add_argument( 575 "--setpref", 576 action="append", 577 dest="extra_prefs", 578 metavar="<pref>=<value>", 579 help="Defines additional user preferences.", 580 ) 581 p.add_argument( 582 "--setopt", 583 action="append", 584 dest="extra_options", 585 metavar="<option>=<value>", 586 help="Defines additional options for `puppeteer.launch`.", 587 ) 588 p.add_argument( 589 "--this-chunk", 590 type=str, 591 default="1", 592 help="Defines a current chunk to run.", 593 ) 594 p.add_argument( 595 "--total-chunks", 596 type=str, 597 default="1", 598 help="Defines a total amount of chunks to run.", 599 ) 600 p.add_argument( 601 "-v", 602 dest="verbosity", 603 action="count", 604 default=0, 605 help="Increase remote agent logging verbosity to include " 606 "debug level messages with -v, trace messages with -vv," 607 "and to not truncate long trace messages with -vvv", 608 ) 609 p.add_argument("tests", nargs="*") 610 mozlog.commandline.add_logging_group(p) 611 return p 612 613 614 def is_relevant_expectation( 615 expectation, expected_product, is_headless, expected_platform 616 ): 617 parameters = expectation["parameters"] 618 619 if expected_product == "firefox": 620 is_expected_product = ( 621 "chrome" not in parameters and "chrome-headless-shell" not in parameters 622 ) 623 else: 624 is_expected_product = "firefox" not in parameters 625 626 is_expected_protocol = "cdp" not in parameters 627 628 if is_headless == "True": 629 is_expected_mode = "headful" not in parameters 630 else: 631 is_expected_mode = "headless" not in parameters 632 633 is_expected_platform = expected_platform in expectation["platforms"] 634 635 return ( 636 is_expected_product 637 and is_expected_protocol 638 and is_expected_mode 639 and is_expected_platform 640 ) 641 642 643 @Command( 644 "puppeteer-test", 645 category="testing", 646 description="Run Puppeteer unit tests.", 647 parser=create_parser_puppeteer, 648 ) 649 @CommandArgument( 650 "--no-install", 651 dest="install", 652 action="store_false", 653 default=True, 654 help="Do not install the Puppeteer package", 655 ) 656 def puppeteer_test( 657 command_context, 658 binary=None, 659 ci=False, 660 disable_fission=False, 661 enable_webrender=False, 662 headless=False, 663 extra_prefs=None, 664 extra_options=None, 665 install=False, 666 verbosity=0, 667 tests=None, 668 product="firefox", 669 this_chunk="1", 670 total_chunks="1", 671 **kwargs, 672 ): 673 logger = mozlog.commandline.setup_logging( 674 "puppeteer-test", kwargs, {"mach": sys.stdout} 675 ) 676 677 # moztest calls this programmatically with test objects or manifests 678 if "test_objects" in kwargs and tests is not None: 679 logger.error("Expected either 'test_objects' or 'tests'") 680 exit(1) 681 682 if product != "firefox" and extra_prefs is not None: 683 logger.error("User preferences are not recognized by %s" % product) 684 exit(1) 685 686 if "test_objects" in kwargs: 687 tests = [] 688 for test in kwargs["test_objects"]: 689 tests.append(test["path"]) 690 691 prefs = {} 692 for s in extra_prefs or []: 693 kv = s.split("=") 694 if len(kv) != 2: 695 logger.error(f"syntax error in --setpref={s}") 696 exit(EX_USAGE) 697 prefs[kv[0]] = kv[1].strip() 698 699 options = {} 700 for s in extra_options or []: 701 kv = s.split("=") 702 if len(kv) != 2: 703 logger.error(f"syntax error in --setopt={s}") 704 exit(EX_USAGE) 705 options[kv[0]] = kv[1].strip() 706 707 prefs.update({"fission.autostart": True}) 708 if disable_fission: 709 prefs.update({"fission.autostart": False}) 710 711 if verbosity == 1: 712 prefs["remote.log.level"] = "Debug" 713 elif verbosity > 1: 714 prefs["remote.log.level"] = "Trace" 715 if verbosity > 2: 716 prefs["remote.log.truncate"] = False 717 718 if install: 719 install_puppeteer(command_context, product, ci) 720 721 params = { 722 "binary": binary, 723 "headless": headless, 724 "enable_webrender": enable_webrender, 725 "extra_prefs": prefs, 726 "product": product, 727 "extra_launcher_options": options, 728 "this_chunk": this_chunk, 729 "total_chunks": total_chunks, 730 } 731 puppeteer = command_context._spawn(PuppeteerRunner) 732 try: 733 return puppeteer.run_test(logger, *tests, **params) 734 except BinaryNotFoundException as e: 735 logger.error(e) 736 logger.info(e.help()) 737 exit(1) 738 except Exception as e: 739 exit(EX_SOFTWARE, e) 740 741 742 def install_puppeteer(command_context, product, ci): 743 setup() 744 745 env = { 746 "CI": "1", # Force the quiet logger of wireit 747 "HUSKY": "0", # Disable any hook checks 748 } 749 750 puppeteer_dir = os.path.join("remote", "test", "puppeteer") 751 puppeteer_dir_full_path = os.path.join(command_context.topsrcdir, puppeteer_dir) 752 puppeteer_test_dir = os.path.join(puppeteer_dir, "test") 753 754 if product == "chrome": 755 env["PUPPETEER_PRODUCT"] = "chrome" 756 env["PUPPETEER_CACHE_DIR"] = os.path.join( 757 command_context.topobjdir, "_tests", puppeteer_dir, ".cache" 758 ) 759 else: 760 env["PUPPETEER_SKIP_DOWNLOAD"] = "1" 761 762 if not ci: 763 run_npm( 764 "run", 765 "clean", 766 cwd=puppeteer_dir_full_path, 767 env=env, 768 exit_on_fail=False, 769 ) 770 771 # Always use the `ci` command to not get updated sub-dependencies installed. 772 run_npm("ci", cwd=puppeteer_dir_full_path, env=env) 773 774 # Build Puppeteer and the code to download browsers. 775 run_npm( 776 "run", 777 "build", 778 cwd=os.path.join(command_context.topsrcdir, puppeteer_test_dir), 779 env=env, 780 ) 781 782 # Run post install steps, including downloading the Chrome browser if requested 783 run_npm("run", "postinstall", cwd=puppeteer_dir_full_path, env=env) 784 785 786 def exit(code, error=None): 787 if error is not None: 788 if isinstance(error, Exception): 789 import traceback 790 791 traceback.print_exc() 792 else: 793 message = str(error).split("\n")[0].strip() 794 print(f"{sys.argv[0]}: {message}", file=sys.stderr) 795 sys.exit(code)