mozharness.py (12528B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 6 Support for running jobs via mozharness. Ideally, most stuff gets run this 7 way, and certainly anything using mozharness should use this approach. 8 9 """ 10 11 from textwrap import dedent 12 13 from mozpack import path as mozpath 14 from taskgraph.util import json 15 from taskgraph.util.schema import Schema 16 from voluptuous import Any, Optional, Required 17 from voluptuous.validators import Match 18 19 from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using 20 from gecko_taskgraph.transforms.job.common import ( 21 docker_worker_add_artifacts, 22 generic_worker_add_artifacts, 23 get_expiration, 24 setup_secrets, 25 ) 26 from gecko_taskgraph.util.attributes import is_try 27 28 mozharness_run_schema = Schema({ 29 Required("using"): "mozharness", 30 # the mozharness script used to run this task, relative to the testing/ 31 # directory and using forward slashes even on Windows 32 Required("script"): str, 33 # Additional paths to look for mozharness configs in. These should be 34 # relative to the base of the source checkout 35 Optional("config-paths"): [str], 36 # the config files required for the task, relative to 37 # testing/mozharness/configs or one of the paths specified in 38 # `config-paths` and using forward slashes even on Windows 39 Required("config"): [str], 40 # any additional actions to pass to the mozharness command 41 Optional("actions"): [ 42 Match("^[a-z0-9-]+$", "actions must be `-` seperated alphanumeric strings") 43 ], 44 # any additional options (without leading --) to be passed to mozharness 45 Optional("options"): [ 46 Match( 47 "^[a-z0-9-]+(=[^ ]+)?$", 48 "options must be `-` seperated alphanumeric strings (with optional argument)", 49 ) 50 ], 51 # --custom-build-variant-cfg value 52 Optional("custom-build-variant-cfg"): str, 53 # Extra configuration options to pass to mozharness. 54 Optional("extra-config"): dict, 55 # If not false, tooltool downloads will be enabled via relengAPIProxy 56 # for either just public files, or all files. Not supported on Windows 57 Required("tooltool-downloads"): Any( 58 False, 59 "public", 60 "internal", 61 ), 62 # The set of secret names to which the task has access; these are prefixed 63 # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting 64 # this will enable any worker features required and set the task's scopes 65 # appropriately. `true` here means ['*'], all secrets. Not supported on 66 # Windows 67 Required("secrets"): Any(bool, [str]), 68 # If true, taskcluster proxy will be enabled; note that it may also be enabled 69 # automatically e.g., for secrets support. Not supported on Windows. 70 Required("taskcluster-proxy"): bool, 71 # If false, indicate that builds should skip producing artifacts. Not 72 # supported on Windows. 73 Required("keep-artifacts"): bool, 74 # If specified, use the in-tree job script specified. 75 Optional("job-script"): str, 76 Required("requires-signed-builds"): bool, 77 # Whether or not to use caches. 78 Optional("use-caches"): Any(bool, [str]), 79 # If false, don't set MOZ_SIMPLE_PACKAGE_NAME 80 # Only disableable on windows 81 Required("use-simple-package"): bool, 82 # If false don't pass --branch mozharness script 83 # Only disableable on windows 84 Required("use-magic-mh-args"): bool, 85 # if true, perform a checkout of a comm-central based branch inside the 86 # gecko checkout 87 Required("comm-checkout"): bool, 88 # Base work directory used to set up the task. 89 Optional("workdir"): str, 90 Optional("run-as-root"): bool, 91 }) 92 93 94 mozharness_defaults = { 95 "tooltool-downloads": False, 96 "secrets": False, 97 "taskcluster-proxy": False, 98 "keep-artifacts": True, 99 "requires-signed-builds": False, 100 "use-simple-package": True, 101 "use-magic-mh-args": True, 102 "comm-checkout": False, 103 "run-as-root": False, 104 } 105 106 107 @run_job_using( 108 "docker-worker", 109 "mozharness", 110 schema=mozharness_run_schema, 111 defaults=mozharness_defaults, 112 ) 113 def mozharness_on_docker_worker_setup(config, job, taskdesc): 114 run = job["run"] 115 116 worker = taskdesc["worker"] = job["worker"] 117 118 if not run.pop("use-simple-package", None): 119 raise NotImplementedError( 120 "Simple packaging cannot be disabled via" 121 "'use-simple-package' on docker-workers" 122 ) 123 if not run.pop("use-magic-mh-args", None): 124 raise NotImplementedError( 125 "Cannot disabled mh magic arg passing via" 126 "'use-magic-mh-args' on docker-workers" 127 ) 128 129 # Running via mozharness assumes an image that contains build.sh: 130 # by default, debian12-amd64-build, but it could be another image (like 131 # android-build). 132 worker.setdefault("docker-image", {"in-tree": "debian12-amd64-build"}) 133 134 worker.setdefault("artifacts", []).append({ 135 "name": "public/logs", 136 "path": "{workdir}/logs/".format(**run), 137 "type": "directory", 138 "expires-after": get_expiration(config, "medium"), 139 }) 140 worker["taskcluster-proxy"] = run.pop("taskcluster-proxy", None) 141 docker_worker_add_artifacts(config, job, taskdesc) 142 143 env = worker.setdefault("env", {}) 144 env.update({ 145 "WORKSPACE": "{workdir}/workspace".format(**run), 146 "MOZHARNESS_CONFIG": " ".join(run.pop("config")), 147 "MOZHARNESS_SCRIPT": run.pop("script"), 148 "MH_BRANCH": config.params["project"], 149 "PYTHONUNBUFFERED": "1", 150 }) 151 152 worker.setdefault("required-volumes", []).append(env["WORKSPACE"]) 153 154 if "actions" in run: 155 env["MOZHARNESS_ACTIONS"] = " ".join(run.pop("actions")) 156 157 if "options" in run: 158 env["MOZHARNESS_OPTIONS"] = " ".join(run.pop("options")) 159 160 if "config-paths" in run: 161 env["MOZHARNESS_CONFIG_PATHS"] = " ".join(run.pop("config-paths")) 162 163 if "custom-build-variant-cfg" in run: 164 env["MH_CUSTOM_BUILD_VARIANT_CFG"] = run.pop("custom-build-variant-cfg") 165 166 extra_config = run.pop("extra-config", {}) 167 extra_config["objdir"] = "obj-build" 168 env["EXTRA_MOZHARNESS_CONFIG"] = json.dumps(extra_config, sort_keys=True) 169 170 if "job-script" in run: 171 env["JOB_SCRIPT"] = run["job-script"] 172 173 if is_try(config.params): 174 env["TRY_COMMIT_MSG"] = config.params["message"] 175 176 # if we're not keeping artifacts, set some env variables to empty values 177 # that will cause the build process to skip copying the results to the 178 # artifacts directory. This will have no effect for operations that are 179 # not builds. 180 if not run.pop("keep-artifacts"): 181 env["DIST_TARGET_UPLOADS"] = "" 182 env["DIST_UPLOADS"] = "" 183 184 # Retry if mozharness returns TBPL_RETRY 185 worker["retry-exit-status"] = [4] 186 187 setup_secrets(config, job, taskdesc) 188 189 run["using"] = "run-task" 190 run["command"] = mozpath.join( 191 "${GECKO_PATH}", 192 run.pop("job-script", "taskcluster/scripts/builder/build-linux.sh"), 193 ) 194 run.pop("secrets") 195 run.pop("requires-signed-builds") 196 197 configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) 198 199 200 @run_job_using( 201 "generic-worker", 202 "mozharness", 203 schema=mozharness_run_schema, 204 defaults=mozharness_defaults, 205 ) 206 def mozharness_on_generic_worker(config, job, taskdesc): 207 assert job["worker"]["os"] in ( 208 "windows", 209 "macosx", 210 ), "only supports windows and macOS right now: {}".format(job["label"]) 211 212 run = job["run"] 213 214 # fail if invalid run options are included 215 invalid = [] 216 if not run.pop("keep-artifacts", True): 217 invalid.append("keep-artifacts") 218 if invalid: 219 raise Exception( 220 "Jobs run using mozharness on Windows do not support properties " 221 + ", ".join(invalid) 222 ) 223 224 worker = taskdesc["worker"] = job["worker"] 225 226 worker["taskcluster-proxy"] = run.pop("taskcluster-proxy", None) 227 228 setup_secrets(config, job, taskdesc) 229 230 taskdesc["worker"].setdefault("artifacts", []).append({ 231 "name": "public/logs", 232 "path": "logs", 233 "type": "directory", 234 "expires-after": get_expiration(config, "medium"), 235 }) 236 237 if not worker.get("skip-artifacts", False): 238 generic_worker_add_artifacts(config, job, taskdesc) 239 240 env = worker.setdefault("env", {}) 241 env.update({ 242 "MH_BRANCH": config.params["project"], 243 }) 244 if run.pop("use-simple-package"): 245 env.update({"MOZ_SIMPLE_PACKAGE_NAME": "target"}) 246 247 extra_config = run.pop("extra-config", {}) 248 extra_config["objdir"] = "obj-build" 249 env["EXTRA_MOZHARNESS_CONFIG"] = json.dumps(extra_config, sort_keys=True) 250 251 # The windows generic worker uses batch files to pass environment variables 252 # to commands. Setting a variable to empty in a batch file unsets, so if 253 # there is no `TRY_COMMIT_MESSAGE`, pass a space instead, so that 254 # mozharness doesn't try to find the commit message on its own. 255 if is_try(config.params): 256 env["TRY_COMMIT_MSG"] = config.params["message"] or "no commit message" 257 258 if not job["attributes"]["build_platform"].startswith(("win", "macosx")): 259 raise Exception( 260 "Task generation for mozharness build jobs currently only supported on " 261 "Windows and macOS" 262 ) 263 264 mh_command = [] 265 if job["worker"]["os"] == "windows": 266 system_python_dir = "c:/mozilla-build/python3/" 267 gecko_path = "%GECKO_PATH%" 268 else: 269 system_python_dir = "" 270 gecko_path = "$GECKO_PATH" 271 272 if run.get("use-python", "system") == "system": 273 python_bindir = system_python_dir 274 else: 275 # $MOZ_PYTHON_HOME is going to be substituted in run-task, when we 276 # know the actual MOZ_PYTHON_HOME value. 277 is_windows = job["worker"]["os"] == "windows" 278 if is_windows: 279 python_bindir = "%MOZ_PYTHON_HOME%/" 280 else: 281 python_bindir = "${MOZ_PYTHON_HOME}/bin/" 282 283 mh_command = [f"{python_bindir}python3"] 284 285 mh_command += [ 286 f"{gecko_path}/mach", 287 "python", 288 "{}/testing/{}".format(gecko_path, run.pop("script")), 289 ] 290 291 for path in run.pop("config-paths", []): 292 mh_command.append(f"--extra-config-path {gecko_path}/{path}") 293 294 for cfg in run.pop("config"): 295 mh_command.extend(("--config", cfg)) 296 if run.pop("use-magic-mh-args"): 297 mh_command.extend(("--branch", config.params["project"])) 298 if job["worker"]["os"] == "windows": 299 mh_command.extend(("--work-dir", r"%cd:Z:=z:%\workspace")) 300 for action in run.pop("actions", []): 301 mh_command.append("--" + action) 302 303 for option in run.pop("options", []): 304 mh_command.append("--" + option) 305 if run.get("custom-build-variant-cfg"): 306 mh_command.append("--custom-build-variant") 307 mh_command.append(run.pop("custom-build-variant-cfg")) 308 309 if job["worker"]["os"] == "macosx": 310 # Ideally, we'd use mozshellutil.quote, but that would single-quote 311 # $GECKO_PATH, which would defeat having the variable in the command 312 # in the first place, as it wouldn't be expanded. 313 # In practice, arguments are expected not to contain characters that 314 # would require quoting. 315 mh_command = " ".join(mh_command) 316 317 run["using"] = "run-task" 318 run["command"] = mh_command 319 run.pop("secrets") 320 run.pop("requires-signed-builds") 321 run.pop("job-script", None) 322 configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) 323 324 # Everything past this point is Windows-specific. 325 if job["worker"]["os"] == "macosx": 326 return 327 328 if taskdesc.get("use-sccache"): 329 worker["command"] = [ 330 # Make the comment part of the first command, as it will help users to 331 # understand what is going on, and why these steps are implemented. 332 dedent( 333 """\ 334 :: sccache currently uses the full compiler commandline as input to the 335 :: cache hash key, so create a symlink to the task dir and build from 336 :: the symlink dir to get consistent paths. 337 if exist z:\\build rmdir z:\\build""" 338 ), 339 r"mklink /d z:\build %cd%", 340 # Grant delete permission on the link to everyone. 341 r"icacls z:\build /grant *S-1-1-0:D /L", 342 r"cd /d z:\build", 343 ] + worker["command"]