bootstrap.py (5907B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 from taskgraph.transforms.base import TransformSequence 7 from taskgraph.util.schema import Schema 8 from voluptuous import Any, Optional, Required 9 10 from gecko_taskgraph.transforms.task import task_description_schema 11 12 transforms = TransformSequence() 13 14 bootstrap_schema = Schema({ 15 # Name of the bootstrap task. 16 Required("name"): str, 17 # Name of the docker image. Ideally, we'd also have tasks for mac and windows, 18 # but we unfortunately don't have workers barebones enough for such testing 19 # to be satisfactory. 20 Required("image"): Any(str, {"in-tree": str}), 21 # Initialization commands. 22 Required("pre-commands"): [str], 23 # relative path (from config.path) to the file task was defined in 24 Optional("task-from"): str, 25 Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], 26 }) 27 28 29 transforms.add_validate(bootstrap_schema) 30 31 32 @transforms.add 33 def bootstrap_tasks(config, tasks): 34 for task in tasks: 35 name = task.pop("name") 36 image = task.pop("image") 37 pre_commands = task.pop("pre-commands") 38 39 head_repo = config.params["head_repository"] 40 head_rev = config.params["head_rev"] 41 42 # Get all the non macos/windows local toolchains (the only ones bootstrap can use), 43 # and use them as dependencies for the tasks we create, so that they don't start 44 # before any potential toolchain task that would be triggered on the same push 45 # (which would lead to bootstrap failing). 46 dependencies = { 47 name: name 48 for name, task in config.kind_dependencies_tasks.items() 49 if task.attributes.get("local-toolchain") 50 and not name.startswith(("toolchain-macos", "toolchain-win")) 51 } 52 # We don't test the artifacts variants, or js, because they are essentially subsets. 53 # Mobile and browser are different enough to warrant testing them separately. 54 for app in ("browser", "mobile_android"): 55 commands = pre_commands + [ 56 # MOZ_AUTOMATION changes the behavior, and we want something closer to user 57 # machines. 58 "unset MOZ_AUTOMATION", 59 f"curl --retry 5 -L -f -O {head_repo}/raw-file/{head_rev}/python/mozboot/bin/bootstrap.py", 60 # We keep using git-cinnabar here because we rely on being able to pull 61 # the head revision from Mercurial. 62 f"python3 bootstrap.py --vcs=git-cinnabar --no-interactive --application-choice {app}", 63 "cd mozilla-unified", 64 # After bootstrap, configure should go through without its own auto-bootstrap. 65 "./mach configure --enable-bootstrap=no-update", 66 # Then a build should go through too. 67 "./mach build", 68 ] 69 70 os_specific = [] 71 if app == "mobile_android": 72 os_specific += ["android*"] 73 for os, filename in ( 74 ("debian", "debian.py"), 75 ("ubuntu", "debian.py"), 76 ("fedora", "centosfedora.py"), 77 ("rockylinux", "centosfedora.py"), 78 ("opensuse", "opensuse.py"), 79 ("gentoo", "gentoo.py"), 80 ("archlinux", "archlinux.py"), 81 ("voidlinux", "void.py"), 82 ): 83 if name.startswith(os): 84 os_specific.append(filename) 85 break 86 else: 87 raise Exception(f"Missing OS specific bootstrap file for {name}") 88 89 taskdesc = { 90 "label": f"{config.kind}-{name}-{app}", 91 "description": f"Bootstrap {app} build on {name}", 92 "always-target": True, 93 "scopes": [], 94 "treeherder": { 95 "symbol": f"Boot({name})", 96 "platform": { 97 "browser": "linux64/opt", 98 "mobile_android": "android-5-0-armv7/opt", 99 }[app], 100 "kind": "other", 101 "tier": 2, 102 }, 103 "run-on-projects": ["trunk"], 104 "run-on-repo-type": task.get("run-on-repo-type", ["git", "hg"]), 105 "worker-type": "b-linux", 106 "worker": { 107 "implementation": "docker-worker", 108 "docker-image": image, 109 "os": "linux", 110 "env": { 111 "GECKO_HEAD_REPOSITORY": head_repo, 112 "GECKO_HEAD_REV": head_rev, 113 "MACH_NO_TERMINAL_FOOTER": "1", 114 "MOZ_SCM_LEVEL": config.params["level"], 115 }, 116 "command": ["sh", "-c", "-x", "-e", " && ".join(commands)], 117 "max-run-time": 7200, 118 }, 119 "dependencies": dependencies, 120 "optimization": { 121 "skip-unless-changed": [ 122 "python/mozboot/bin/bootstrap.py", 123 "python/mozboot/mozboot/base.py", 124 "python/mozboot/mozboot/bootstrap.py", 125 "python/mozboot/mozboot/linux_common.py", 126 "python/mozboot/mozboot/mach_commands.py", 127 "python/mozboot/mozboot/mozconfig.py", 128 "python/mozboot/mozboot/rust.py", 129 "python/mozboot/mozboot/sccache.py", 130 "python/mozboot/mozboot/util.py", 131 ] 132 + [f"python/mozboot/mozboot/{f}" for f in os_specific] 133 }, 134 } 135 136 yield taskdesc