command_util.py (22166B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 import argparse 6 import ast 7 import difflib 8 import errno 9 import importlib.metadata 10 import shlex 11 import sys 12 import types 13 import uuid 14 from collections.abc import Iterable 15 from pathlib import Path 16 from typing import Optional, Union 17 18 from mozfile import load_source 19 20 from .base import MissingFileError, UnknownCommandError 21 22 INVALID_ENTRY_POINT = r""" 23 Entry points should return a list of command providers or directories 24 containing command providers. The following entry point is invalid: 25 26 %s 27 28 You are seeing this because there is an error in an external module attempting 29 to implement a mach command. Please fix the error, or uninstall the module from 30 your system. 31 """.lstrip() 32 33 34 class MachCommandReference: 35 """A reference to a mach command. 36 37 Holds the metadata for a mach command. 38 """ 39 40 module: Path 41 42 def __init__( 43 self, 44 module: Union[str, Path], 45 command_dependencies: Optional[list] = None, 46 ): 47 self.module = Path(module) 48 self.command_dependencies = command_dependencies or [] 49 50 51 MACH_COMMANDS = { 52 "adb": MachCommandReference("mobile/android/mach_commands.py"), 53 "addstory": MachCommandReference("toolkit/content/widgets/mach_commands.py"), 54 "addtest": MachCommandReference("testing/mach_commands.py"), 55 "addwidget": MachCommandReference("toolkit/content/widgets/mach_commands.py"), 56 "android": MachCommandReference("mobile/android/mach_commands.py"), 57 "android-emulator": MachCommandReference("mobile/android/mach_commands.py"), 58 "android-test": MachCommandReference("testing/android-test/mach_commands.py"), 59 "artifact": MachCommandReference( 60 "python/mozbuild/mozbuild/artifact_commands.py", 61 ), 62 "awsy-test": MachCommandReference("testing/awsy/mach_commands.py"), 63 "bootstrap": MachCommandReference( 64 "python/mozboot/mozboot/mach_commands.py", 65 ), 66 "browsertime": MachCommandReference("tools/browsertime/mach_commands.py"), 67 "build": MachCommandReference( 68 "python/mozbuild/mozbuild/build_commands.py", 69 ), 70 "build-backend": MachCommandReference( 71 "python/mozbuild/mozbuild/build_commands.py", 72 ), 73 "buildsymbols": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 74 "buildtokens": MachCommandReference("toolkit/content/widgets/mach_commands.py"), 75 "busted": MachCommandReference("tools/mach_commands.py"), 76 "cargo": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 77 "clang-format": MachCommandReference( 78 "python/mozbuild/mozbuild/code_analysis/mach_commands.py" 79 ), 80 "clang-tidy": MachCommandReference( 81 "python/mozbuild/mozbuild/code_analysis/mach_commands.py" 82 ), 83 "clobber": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 84 "compare-locales": MachCommandReference("tools/compare-locales/mach_commands.py"), 85 "compileflags": MachCommandReference( 86 "python/mozbuild/mozbuild/compilation/codecomplete.py" 87 ), 88 "configure": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"), 89 "cppunittest": MachCommandReference("testing/mach_commands.py"), 90 "crash-ping-metrics": MachCommandReference( 91 "toolkit/crashreporter/crashping/glean_metrics.py" 92 ), 93 "crashtest": MachCommandReference("layout/tools/reftest/mach_commands.py"), 94 "data-review": MachCommandReference( 95 "toolkit/components/glean/build_scripts/mach_commands.py" 96 ), 97 "devtools-node-test": MachCommandReference("devtools/mach_commands.py"), 98 "doc": MachCommandReference("tools/moztreedocs/mach_commands.py"), 99 "doctor": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 100 "environment": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 101 "eslint": MachCommandReference("tools/lint/mach_commands.py"), 102 "event-into-legacy": MachCommandReference( 103 "toolkit/components/glean/build_scripts/mach_commands.py" 104 ), 105 "fetch-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"), 106 "file-info": MachCommandReference( 107 "python/mozbuild/mozbuild/frontend/mach_commands.py" 108 ), 109 "firefox-ui-functional": MachCommandReference( 110 "testing/firefox-ui/mach_commands.py" 111 ), 112 "fluent-migration-test": MachCommandReference("testing/mach_commands.py"), 113 "format": MachCommandReference("tools/lint/mach_commands.py"), 114 "geckodriver": MachCommandReference("testing/geckodriver/mach_commands.py"), 115 "geckoview-junit": MachCommandReference( 116 "testing/mochitest/mach_commands.py", ["test"] 117 ), 118 "gen-uuid": MachCommandReference("dom/base/mach_commands.py"), 119 "gen-use-counter-metrics": MachCommandReference("dom/base/mach_commands.py"), 120 "generate-python-lockfiles": MachCommandReference( 121 "python/mozbuild/mozbuild/lockfiles/mach_commands.py", 122 ), 123 "generate-test-certs": MachCommandReference( 124 "security/manager/tools/mach_commands.py" 125 ), 126 "gifft": MachCommandReference( 127 "toolkit/components/telemetry/build_scripts/mach_commands.py" 128 ), 129 "gecko-trace": MachCommandReference( 130 "toolkit/components/gecko-trace/mach_commands.py" 131 ), 132 "glean": MachCommandReference( 133 "toolkit/components/glean/build_scripts/mach_commands.py" 134 ), 135 "gradle": MachCommandReference("mobile/android/mach_commands.py"), 136 "gradle-install": MachCommandReference("mobile/android/mach_commands.py"), 137 "gtest": MachCommandReference( 138 "python/mozbuild/mozbuild/mach_commands.py", ["test"] 139 ), 140 "hazards": MachCommandReference("js/src/devtools/rootAnalysis/mach_commands.py"), 141 "ide": MachCommandReference("python/mozbuild/mozbuild/backend/mach_commands.py"), 142 "import-pr": MachCommandReference("tools/vcs/mach_commands.py"), 143 "install": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 144 "intermittents": MachCommandReference("testing/intermittents_mach_commands.py"), 145 "install-moz-phab": MachCommandReference("tools/phabricator/mach_commands.py"), 146 "jit-test": MachCommandReference("testing/mach_commands.py"), 147 "jsapi-tests": MachCommandReference("testing/mach_commands.py"), 148 "jsshell-bench": MachCommandReference("testing/mach_commands.py"), 149 "jstestbrowser": MachCommandReference("layout/tools/reftest/mach_commands.py"), 150 "jstests": MachCommandReference("testing/mach_commands.py"), 151 "lint": MachCommandReference("tools/lint/mach_commands.py"), 152 "logspam": MachCommandReference("tools/mach_commands.py"), 153 "mach-commands": MachCommandReference("python/mach/mach/commands/commandinfo.py"), 154 "mach-completion": MachCommandReference("python/mach/mach/commands/commandinfo.py"), 155 "mach-debug-commands": MachCommandReference( 156 "python/mach/mach/commands/commandinfo.py" 157 ), 158 "macos-sign": MachCommandReference("tools/signing/macos/mach_commands.py"), 159 "manifest": MachCommandReference("testing/manifest/mach_commands.py"), 160 "platform-diff": MachCommandReference("testing/mach_commands.py"), 161 "marionette-test": MachCommandReference("testing/marionette/mach_commands.py"), 162 "mochitest": MachCommandReference("testing/mochitest/mach_commands.py", ["test"]), 163 "mots": MachCommandReference("tools/mach_commands.py"), 164 "mozbuild-reference": MachCommandReference( 165 "python/mozbuild/mozbuild/frontend/mach_commands.py", 166 ), 167 "mozharness": MachCommandReference("testing/mozharness/mach_commands.py"), 168 "mozregression": MachCommandReference("tools/mach_commands.py"), 169 "newtab": MachCommandReference("browser/extensions/newtab/mach_commands.py"), 170 "node": MachCommandReference("tools/mach_commands.py"), 171 "npm": MachCommandReference("tools/mach_commands.py"), 172 "nss-uplift": MachCommandReference("security/mach_commands.py"), 173 "package": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 174 "package-multi-locale": MachCommandReference( 175 "python/mozbuild/mozbuild/mach_commands.py" 176 ), 177 "pastebin": MachCommandReference("tools/mach_commands.py"), 178 "perf-data-review": MachCommandReference( 179 "toolkit/components/glean/build_scripts/mach_commands.py" 180 ), 181 "perftest": MachCommandReference("python/mozperftest/mozperftest/mach_commands.py"), 182 "perftest-test": MachCommandReference( 183 "python/mozperftest/mozperftest/mach_commands.py", 184 ), 185 "perftest-tools": MachCommandReference( 186 "python/mozperftest/mozperftest/mach_commands.py" 187 ), 188 "power": MachCommandReference("tools/power/mach_commands.py"), 189 "prettier": MachCommandReference("tools/lint/mach_commands.py"), 190 "puppeteer-test": MachCommandReference("remote/mach_commands.py"), 191 "python": MachCommandReference("python/mach_commands.py"), 192 "python-test": MachCommandReference("python/mach_commands.py"), 193 "raptor": MachCommandReference("testing/raptor/mach_commands.py"), 194 "raptor-test": MachCommandReference("testing/raptor/mach_commands.py"), 195 "reftest": MachCommandReference("layout/tools/reftest/mach_commands.py"), 196 "release": MachCommandReference("python/mozrelease/mozrelease/mach_commands.py"), 197 "release-history": MachCommandReference("taskcluster/mach_commands.py"), 198 "remote": MachCommandReference("remote/mach_commands.py"), 199 "repackage": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 200 "repackage-single-locales": MachCommandReference( 201 "python/mozbuild/mozbuild/mach_commands.py" 202 ), 203 "resource-usage": MachCommandReference( 204 "python/mozbuild/mozbuild/build_commands.py", 205 ), 206 "run": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 207 "run-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"), 208 "rusttests": MachCommandReference("testing/mach_commands.py"), 209 "settings": MachCommandReference("python/mach/mach/commands/settings.py"), 210 "show-log": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 211 "static-analysis": MachCommandReference( 212 "python/mozbuild/mozbuild/code_analysis/mach_commands.py" 213 ), 214 "storybook": MachCommandReference( 215 "browser/components/storybook/mach_commands.py", ["run"] 216 ), 217 "talos-test": MachCommandReference("testing/talos/mach_commands.py"), 218 "taskgraph": MachCommandReference("taskcluster/mach_commands.py"), 219 "telemetry-tests-client": MachCommandReference( 220 "toolkit/components/telemetry/tests/marionette/mach_commands.py" 221 ), 222 "test": MachCommandReference("testing/mach_commands.py"), 223 "test-info": MachCommandReference("testing/mach_commands.py"), 224 "test-interventions": MachCommandReference( 225 "testing/webcompat/mach_commands.py", 226 ), 227 "tps-build": MachCommandReference("testing/tps/mach_commands.py"), 228 "try": MachCommandReference("tools/tryselect/mach_commands.py"), 229 "ts": MachCommandReference("tools/ts/mach_commands.py"), 230 "uniffi": MachCommandReference( 231 "toolkit/components/uniffi-bindgen-gecko-js/mach_commands.py" 232 ), 233 "update": MachCommandReference("tools/update-programs/mach_commands.py"), 234 "update-glean": MachCommandReference( 235 "toolkit/components/glean/build_scripts/mach_commands.py" 236 ), 237 "update-glean-tags": MachCommandReference( 238 "toolkit/components/glean/build_scripts/mach_commands.py" 239 ), 240 "update-test": MachCommandReference("testing/update/mach_commands.py"), 241 "use-moz-src": MachCommandReference("tools/use-moz-src/mach_commands.py"), 242 "valgrind-test": MachCommandReference("build/valgrind/mach_commands.py"), 243 "vcs-setup": MachCommandReference( 244 "python/mozboot/mozboot/mach_commands.py", 245 ), 246 "vendor": MachCommandReference( 247 "python/mozbuild/mozbuild/vendor/mach_commands.py", 248 ), 249 "warnings-list": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), 250 "warnings-summary": MachCommandReference( 251 "python/mozbuild/mozbuild/mach_commands.py" 252 ), 253 "watch": MachCommandReference( 254 "python/mozbuild/mozbuild/mach_commands.py", 255 ), 256 "web-platform-tests": MachCommandReference( 257 "testing/web-platform/mach_commands.py", 258 ), 259 "web-platform-tests-update": MachCommandReference( 260 "testing/web-platform/mach_commands.py", 261 ), 262 "webidl-example": MachCommandReference("dom/bindings/mach_commands.py"), 263 "webidl-parser-test": MachCommandReference("dom/bindings/mach_commands.py"), 264 "wpt": MachCommandReference("testing/web-platform/mach_commands.py"), 265 "wpt-fetch-logs": MachCommandReference("testing/web-platform/mach_commands.py"), 266 "wpt-interop-score": MachCommandReference("testing/web-platform/mach_commands.py"), 267 "wpt-manifest-update": MachCommandReference( 268 "testing/web-platform/mach_commands.py" 269 ), 270 "wpt-metadata-merge": MachCommandReference("testing/web-platform/mach_commands.py"), 271 "wpt-metadata-summary": MachCommandReference( 272 "testing/web-platform/mach_commands.py" 273 ), 274 "wpt-serve": MachCommandReference("testing/web-platform/mach_commands.py"), 275 "wpt-test-paths": MachCommandReference("testing/web-platform/mach_commands.py"), 276 "wpt-unittest": MachCommandReference("testing/web-platform/mach_commands.py"), 277 "wpt-update": MachCommandReference("testing/web-platform/mach_commands.py"), 278 "xpcshell": MachCommandReference("js/xpconnect/mach_commands.py"), 279 "xpcshell-test": MachCommandReference( 280 "testing/xpcshell/mach_commands.py", ["test"] 281 ), 282 } 283 284 285 class DecoratorVisitor(ast.NodeVisitor): 286 def __init__(self): 287 self.results = {} 288 289 def visit_FunctionDef(self, node): 290 # We only care about `Command` and `SubCommand` decorators, since 291 # they are the only ones that can specify virtualenv_name 292 decorators = [ 293 decorator 294 for decorator in node.decorator_list 295 if isinstance(decorator, ast.Call) 296 and isinstance(decorator.func, ast.Name) 297 and decorator.func.id in ["SubCommand", "Command"] 298 ] 299 300 relevant_kwargs = ["command", "subcommand", "virtualenv_name"] 301 302 for decorator in decorators: 303 kwarg_dict = {} 304 305 for name, arg in zip(["command", "subcommand"], decorator.args): 306 kwarg_dict[name] = arg.value 307 308 for keyword in decorator.keywords: 309 if keyword.arg not in relevant_kwargs: 310 # We only care about these 3 kwargs, so we can safely skip the rest 311 continue 312 313 kwarg_dict[keyword.arg] = keyword.value.value 314 315 command = kwarg_dict.pop("command") 316 self.results.setdefault(command, {}) 317 318 sub_command = kwarg_dict.pop("subcommand", None) 319 virtualenv_name = kwarg_dict.pop("virtualenv_name", None) 320 321 if sub_command: 322 self.results[command].setdefault("subcommands", {}) 323 sub_command_dict = self.results[command]["subcommands"].setdefault( 324 sub_command, {} 325 ) 326 327 if virtualenv_name: 328 sub_command_dict["virtualenv_name"] = virtualenv_name 329 elif virtualenv_name: 330 # If there is no `subcommand` we are in the `@Command` 331 # decorator, and need to store the virtualenv_name for 332 # the 'command'. 333 self.results[command]["virtualenv_name"] = virtualenv_name 334 335 self.generic_visit(node) 336 337 338 def command_virtualenv_info_for_module(module_path): 339 with module_path.open("r", encoding="utf-8") as file: 340 content = file.read() 341 342 tree = ast.parse(content) 343 visitor = DecoratorVisitor() 344 visitor.visit(tree) 345 346 return visitor.results 347 348 349 class DetermineCommandVenvAction(argparse.Action): 350 def __init__( 351 self, 352 option_strings, 353 dest, 354 topsrcdir, 355 required=True, 356 ): 357 self.topsrcdir = topsrcdir 358 argparse.Action.__init__( 359 self, 360 option_strings, 361 dest, 362 required=required, 363 help=argparse.SUPPRESS, 364 nargs=argparse.REMAINDER, 365 ) 366 367 def __call__(self, parser, namespace, values, option_string=None): 368 if len(values) == 0: 369 return 370 371 command = values[0] 372 373 aliases = namespace.mach_command_aliases 374 375 if command in aliases: 376 alias = aliases[command] 377 arg_string = shlex.split(alias) 378 command = arg_string.pop(0) 379 380 # the "help" command does not have a module file, it's handled 381 # a bit later and should be skipped here. 382 if command == "help": 383 return 384 385 command_reference = MACH_COMMANDS.get(command) 386 387 if not command_reference: 388 # Try to find similarly named commands, may raise UnknownCommandError. 389 suggested_command = suggest_command(command) 390 391 sys.stderr.write( 392 f"We're assuming the '{command}' command is '{suggested_command}' and we're executing it for you.\n\n" 393 ) 394 395 command = suggested_command 396 command_reference = MACH_COMMANDS.get(command) 397 398 setattr(namespace, "command_name", command) 399 400 if len(values) > 1: 401 potential_sub_command_name = values[1] 402 else: 403 potential_sub_command_name = None 404 405 module_path = Path(self.topsrcdir) / command_reference.module 406 module_dict = command_virtualenv_info_for_module(module_path) 407 command_dict = module_dict.get(command, {}) 408 409 if not command_dict: 410 return 411 412 site = command_dict.get("virtualenv_name", "common") 413 414 if potential_sub_command_name and not potential_sub_command_name.startswith( 415 "-" 416 ): 417 all_sub_commands_dict = command_dict.get("subcommands", {}) 418 419 if all_sub_commands_dict: 420 sub_command_dict = all_sub_commands_dict.get( 421 potential_sub_command_name, {} 422 ) 423 424 if sub_command_dict: 425 site = sub_command_dict.get("virtualenv_name", "common") 426 427 setattr(namespace, "site_name", site) 428 429 430 def suggest_command(command): 431 names = MACH_COMMANDS.keys() 432 # We first try to look for a valid command that is very similar to the given command. 433 suggested_commands = difflib.get_close_matches(command, names, cutoff=0.8) 434 # If we find more than one matching command, or no command at all, 435 # we give command suggestions instead (with a lower matching threshold). 436 # All commands that start with the given command (for instance: 437 # 'mochitest-plain', 'mochitest-chrome', etc. for 'mochitest-') 438 # are also included. 439 if len(suggested_commands) != 1: 440 suggested_commands = set(difflib.get_close_matches(command, names, cutoff=0.5)) 441 suggested_commands |= {cmd for cmd in names if cmd.startswith(command)} 442 raise UnknownCommandError(command, "run", suggested_commands) 443 444 return suggested_commands[0] 445 446 447 def load_commands_from_directory(path: Path): 448 """Scan for mach commands from modules in a directory. 449 450 This takes a path to a directory, loads the .py files in it, and 451 registers and found mach command providers with this mach instance. 452 """ 453 for f in sorted(path.iterdir()): 454 if not f.suffix == ".py" or f.name == "__init__.py": 455 continue 456 457 full_path = path / f 458 module_name = f"mach.commands.{str(f)[0:-3]}" 459 460 load_commands_from_file(full_path, module_name=module_name) 461 462 463 def load_commands_from_file(path: Union[str, Path], module_name=None): 464 """Scan for mach commands from a file. 465 466 This takes a path to a file and loads it as a Python module under the 467 module name specified. If no name is specified, a random one will be 468 chosen. 469 """ 470 if module_name is None: 471 # Ensure parent module is present otherwise we'll (likely) get 472 # an error due to unknown parent. 473 if "mach.commands" not in sys.modules: 474 mod = types.ModuleType("mach.commands") 475 sys.modules["mach.commands"] = mod 476 477 module_name = f"mach.commands.{uuid.uuid4().hex}" 478 479 try: 480 load_source(module_name, str(path)) 481 except OSError as e: 482 if e.errno != errno.ENOENT: 483 raise 484 485 raise MissingFileError(f"{path} does not exist") 486 487 488 def load_commands_from_spec( 489 spec: dict[str, MachCommandReference], topsrcdir: str, missing_ok=False 490 ): 491 """Load mach commands based on the given spec. 492 493 Takes a dictionary mapping command names to their metadata. 494 """ 495 modules = {spec[command].module for command in spec} 496 497 for path in modules: 498 try: 499 load_commands_from_file(Path(topsrcdir) / path) 500 except MissingFileError: 501 if not missing_ok: 502 raise 503 504 505 def load_commands_from_entry_point(group="mach.providers"): 506 """Scan installed packages for mach command provider entry points. An 507 entry point is a function that returns a list of paths to files or 508 directories containing command providers. 509 510 This takes an optional group argument which specifies the entry point 511 group to use. If not specified, it defaults to 'mach.providers'. 512 """ 513 for entry in importlib.metadata.entry_points(group=group): 514 paths = [Path(path) for path in entry.load()()] 515 if not isinstance(paths, Iterable): 516 print(INVALID_ENTRY_POINT % entry) 517 sys.exit(1) 518 519 for path in paths: 520 if path.is_file(): 521 load_commands_from_file(path) 522 elif path.is_dir(): 523 load_commands_from_directory(path) 524 else: 525 print(f"command provider '{path}' does not exist") 526 527 528 def load_command_module_from_command_name(command_name: str, topsrcdir: str): 529 command_reference = MACH_COMMANDS.get(command_name) 530 load_commands_from_spec( 531 {command_name: command_reference}, topsrcdir, missing_ok=False 532 )