toolchain.py (10470B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Support for running toolchain-building jobs via dedicated scripts 6 """ 7 8 import os 9 10 import taskgraph 11 from mozshellutil import quote as shell_quote 12 from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by 13 from voluptuous import Any, Optional, Required 14 15 from gecko_taskgraph import GECKO 16 from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using 17 from gecko_taskgraph.transforms.job.common import ( 18 docker_worker_add_artifacts, 19 generic_worker_add_artifacts, 20 ) 21 from gecko_taskgraph.util.attributes import RELEASE_PROJECTS 22 from gecko_taskgraph.util.hash import hash_paths 23 24 CACHE_TYPE = "toolchains.v3" 25 26 toolchain_run_schema = Schema({ 27 Required("using"): "toolchain-script", 28 # The script (in taskcluster/scripts/misc) to run. 29 # Python scripts are invoked with `mach python` so vendored libraries 30 # are available. 31 Required("script"): str, 32 # Arguments to pass to the script. 33 Optional("arguments"): [str], 34 # If not false, tooltool downloads will be enabled via relengAPIProxy 35 # for either just public files, or all files. Not supported on Windows 36 Required("tooltool-downloads"): Any( 37 False, 38 "public", 39 "internal", 40 ), 41 # Sparse profile to give to checkout using `run-task`. If given, 42 # Defaults to "toolchain-build". The value is relative to 43 # "sparse-profile-prefix", optionally defined below is the path, 44 # defaulting to "build/sparse-profiles". 45 # i.e. `build/sparse-profiles/toolchain-build`. 46 # If `None`, instructs `run-task` to not use a sparse profile at all. 47 Required("sparse-profile"): Any(str, None), 48 # The relative path to the sparse profile. 49 Optional("sparse-profile-prefix"): str, 50 # Paths/patterns pointing to files that influence the outcome of a 51 # toolchain build. 52 Optional("resources"): [str], 53 # Path to the artifact produced by the toolchain job 54 Required("toolchain-artifact"): str, 55 Optional( 56 "toolchain-alias", 57 description="An alias that can be used instead of the real toolchain job name in " 58 "fetch stanzas for jobs.", 59 ): optionally_keyed_by("project", Any(None, str, [str])), 60 Optional( 61 "toolchain-env", 62 description="Additional env variables to add to the worker when using this toolchain", 63 ): {str: object}, 64 Optional( 65 "toolchain-extract", 66 description="Whether the toolchain should be extracted after it is fetched " 67 + "(default: True)", 68 ): bool, 69 # Base work directory used to set up the task. 70 Optional("workdir"): str, 71 }) 72 73 74 def get_digest_data(config, run, taskdesc): 75 files = list(run.pop("resources", [])) 76 # The script 77 files.append("taskcluster/scripts/misc/{}".format(run["script"])) 78 env = taskdesc["worker"].get("env", {}) 79 # Tooltool manifest if any is defined: 80 tooltool_manifest = env.get("TOOLTOOL_MANIFEST") 81 if tooltool_manifest: 82 files.append(tooltool_manifest) 83 84 # Store resources as an attribute for verification 85 taskdesc.setdefault("attributes", {})["toolchain-resources"] = sorted(files) 86 87 # Accumulate dependency hashes for index generation. 88 data = [hash_paths(GECKO, files)] 89 90 data.append(taskdesc["attributes"]["toolchain-artifact"]) 91 92 # If the task uses an in-tree docker image, we want it to influence 93 # the index path as well. Ideally, the content of the docker image itself 94 # should have an influence, but at the moment, we can't get that 95 # information here. So use the docker image name as a proxy. Not a lot of 96 # changes to docker images actually have an impact on the resulting 97 # toolchain artifact, so we'll just rely on such important changes to be 98 # accompanied with a docker image name change. 99 image = taskdesc["worker"].get("docker-image", {}).get("in-tree") 100 if image: 101 data.append(image) 102 103 # Likewise script arguments should influence the index. 104 args = run.get("arguments") 105 if args: 106 data.extend(args) 107 108 # Environment variables defined by the user (as opposed to added by transforms) 109 for key, value in env.items(): 110 # Ignore the tooltool manifest because its content was already added above. 111 if key == "TOOLTOOL_MANIFEST": 112 continue 113 data.append(f"##{key}={value}##") 114 115 if taskdesc["attributes"].get("rebuild-on-release"): 116 # Add whether this is a release branch or not 117 data.append(str(config.params["project"] in RELEASE_PROJECTS)) 118 return data 119 120 121 def common_toolchain(config, job, taskdesc, is_docker): 122 run = job["run"] 123 124 worker = taskdesc["worker"] = job["worker"] 125 worker["chain-of-trust"] = True 126 127 if is_docker: 128 # If the task doesn't have a docker-image, set a default 129 worker.setdefault("docker-image", {"in-tree": "deb12-toolchain-build"}) 130 131 if job["worker"]["os"] == "windows": 132 # There were no caches on generic-worker before bug 1519472, and they cause 133 # all sorts of problems with Windows toolchain tasks, disable them until 134 # tasks are ready. 135 run["use-caches"] = False 136 137 attributes = taskdesc.setdefault("attributes", {}) 138 attributes["toolchain-artifact"] = run["toolchain-artifact"] 139 toolchain_artifact = attributes["toolchain-artifact"] 140 if not toolchain_artifact.startswith("public/build/"): 141 if "artifact_prefix" in attributes: 142 raise Exception( 143 "Toolchain {} has an artifact_prefix attribute. That is not" 144 " allowed on toolchain tasks.".format(taskdesc["label"]) 145 ) 146 attributes["artifact_prefix"] = os.path.dirname(toolchain_artifact) 147 148 # Note: this must be called before altering `env`. 149 digest_data = get_digest_data(config, run, taskdesc) 150 151 env = worker.setdefault("env", {}) 152 env.update({ 153 "MOZ_BUILD_DATE": config.params["moz_build_date"], 154 "MOZ_SCM_LEVEL": config.params["level"], 155 "TOOLCHAIN_ARTIFACT": run.pop("toolchain-artifact"), 156 }) 157 158 if is_docker: 159 # Toolchain checkouts don't live under {workdir}/checkouts 160 workspace = "{workdir}/workspace/build".format(**run) 161 env["GECKO_PATH"] = f"{workspace}/src" 162 163 resolve_keyed_by( 164 run, 165 "toolchain-alias", 166 item_name=taskdesc["label"], 167 project=config.params["project"], 168 ) 169 alias = run.pop("toolchain-alias", None) 170 if alias: 171 attributes["toolchain-alias"] = alias 172 if "toolchain-env" in run: 173 attributes["toolchain-env"] = run.pop("toolchain-env") 174 if "toolchain-extract" in run: 175 attributes["toolchain-extract"] = run.pop("toolchain-extract") 176 177 # Allow the job to specify where artifacts come from, but add 178 # public/build if it's not there already. 179 artifacts = worker.setdefault("artifacts", []) 180 if not artifacts: 181 if is_docker: 182 docker_worker_add_artifacts(config, job, taskdesc) 183 else: 184 generic_worker_add_artifacts(config, job, taskdesc) 185 186 if job.get("attributes", {}).get("cached_task") is not False and not taskgraph.fast: 187 name = taskdesc["label"].replace(f"{config.kind}-", "", 1) 188 taskdesc["cache"] = { 189 "type": CACHE_TYPE, 190 "name": name, 191 "digest-data": digest_data, 192 } 193 194 # Toolchains that are used for local development need to be built on a 195 # level-3 branch to be installable via `mach bootstrap`. 196 local_toolchain = taskdesc["attributes"].get("local-toolchain") 197 if local_toolchain: 198 if taskdesc.get("run-on-projects"): 199 raise Exception( 200 "Toolchain {} used for local developement must not have" 201 " run-on-projects set".format(taskdesc["label"]) 202 ) 203 taskdesc["run-on-projects"] = ["integration", "release"] 204 205 script = run.pop("script") 206 arguments = run.pop("arguments", []) 207 if local_toolchain and not attributes["toolchain-artifact"].startswith("public/"): 208 # Local toolchains with private artifacts are expected to have a script that 209 # fill a directory given as a final command line argument. That script, and the 210 # arguments provided, are used by the build system bootstrap code, and for the 211 # corresponding CI tasks, the command is wrapped with a script that creates an 212 # artifact based on that filled directory. 213 # We prefer automatic wrapping rather than manual wrapping in the yaml because 214 # it makes the index independent of the wrapper script, which is irrelevant. 215 # Also, an attribute is added for the bootstrap code to be able to easily parse 216 # the command. 217 attributes["toolchain-command"] = { 218 "script": script, 219 "arguments": list(arguments), 220 } 221 arguments.insert(0, script) 222 script = "private_local_toolchain.sh" 223 224 run["using"] = "run-task" 225 if is_docker: 226 gecko_path = "workspace/build/src" 227 elif job["worker"]["os"] == "windows": 228 gecko_path = "%GECKO_PATH%" 229 else: 230 gecko_path = "$GECKO_PATH" 231 232 if is_docker: 233 run["cwd"] = run["workdir"] 234 run["command"] = [f"{gecko_path}/taskcluster/scripts/misc/{script}"] + arguments 235 if not is_docker: 236 # Don't quote the first item in the command because it purposely contains 237 # an environment variable that is not meant to be quoted. 238 if len(run["command"]) > 1: 239 run["command"] = run["command"][0] + " " + shell_quote(*run["command"][1:]) 240 else: 241 run["command"] = run["command"][0] 242 243 configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) 244 245 246 toolchain_defaults = { 247 "tooltool-downloads": False, 248 "sparse-profile": "toolchain-build", 249 } 250 251 252 @run_job_using( 253 "docker-worker", 254 "toolchain-script", 255 schema=toolchain_run_schema, 256 defaults=toolchain_defaults, 257 ) 258 def docker_worker_toolchain(config, job, taskdesc): 259 common_toolchain(config, job, taskdesc, is_docker=True) 260 261 262 @run_job_using( 263 "generic-worker", 264 "toolchain-script", 265 schema=toolchain_run_schema, 266 defaults=toolchain_defaults, 267 ) 268 def generic_worker_toolchain(config, job, taskdesc): 269 common_toolchain(config, job, taskdesc, is_docker=False)