mozharness_test.py (17951B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import os 7 import re 8 9 from taskgraph.util import json 10 from taskgraph.util.schema import Schema 11 from taskgraph.util.taskcluster import get_artifact_path 12 from voluptuous import Extra, Optional, Required 13 14 from gecko_taskgraph.transforms.job import configure_taskdesc_for_run, run_job_using 15 from gecko_taskgraph.transforms.job.common import ( 16 docker_worker_add_artifacts, 17 generic_worker_add_artifacts, 18 get_expiration, 19 support_vcs_checkout, 20 ) 21 from gecko_taskgraph.transforms.test import normpath, test_description_schema 22 from gecko_taskgraph.util.attributes import is_try 23 from gecko_taskgraph.util.chunking import get_test_tags 24 from gecko_taskgraph.util.perftest import is_external_browser 25 26 VARIANTS = [ 27 "shippable", 28 "shippable-qr", 29 "shippable-lite", 30 "shippable-lite-qr", 31 "devedition", 32 "pgo", 33 "asan", 34 "stylo", 35 "qr", 36 "ccov", 37 ] 38 39 40 def get_variant(test_platform): 41 for v in VARIANTS: 42 if f"-{v}/" in test_platform: 43 return v 44 return "" 45 46 47 mozharness_test_run_schema = Schema({ 48 Required("using"): "mozharness-test", 49 Required("test"): { 50 Required("test-platform"): str, 51 Required("mozharness"): test_description_schema["mozharness"], 52 Required("docker-image"): test_description_schema["docker-image"], 53 Required("loopback-video"): test_description_schema["loopback-video"], 54 Required("loopback-audio"): test_description_schema["loopback-audio"], 55 Required("max-run-time"): test_description_schema["max-run-time"], 56 Optional("retry-exit-status"): test_description_schema["retry-exit-status"], 57 Extra: object, 58 }, 59 # Base work directory used to set up the task. 60 Optional("workdir"): str, 61 }) 62 63 64 def test_packages_url(taskdesc): 65 """Account for different platforms that name their test packages differently""" 66 artifact_path = "target.test_packages.json" 67 # for android shippable we need to add 'en-US' to the artifact url 68 test = taskdesc["run"]["test"] 69 if ( 70 "android" in test["test-platform"] 71 and ( 72 get_variant(test["test-platform"]) 73 in ("shippable", "shippable-qr", "shippable-lite", "shippable-lite-qr") 74 ) 75 and not is_external_browser(test.get("try-name", "")) 76 ): 77 artifact_path = os.path.join("en-US", artifact_path) 78 return f"<build/{get_artifact_path(taskdesc, artifact_path)}>" 79 80 81 def installer_url(taskdesc): 82 test = taskdesc["run"]["test"] 83 mozharness = test["mozharness"] 84 85 if "installer-url" in mozharness: 86 return mozharness["installer-url"] 87 upstream_task = "build-signing" if mozharness["requires-signed-builds"] else "build" 88 return f"<{upstream_task}/{mozharness['build-artifact-name']}>" 89 90 91 @run_job_using("docker-worker", "mozharness-test", schema=mozharness_test_run_schema) 92 def mozharness_test_on_docker(config, job, taskdesc): 93 run = job["run"] 94 test = taskdesc["run"]["test"] 95 mozharness = test["mozharness"] 96 worker = taskdesc["worker"] = job["worker"] 97 98 # apply some defaults 99 worker["docker-image"] = test["docker-image"] 100 worker["allow-ptrace"] = True # required for all tests, for crashreporter 101 worker["loopback-video"] = test["loopback-video"] 102 worker["loopback-audio"] = test["loopback-audio"] 103 worker["max-run-time"] = test["max-run-time"] 104 worker["retry-exit-status"] = test["retry-exit-status"] 105 if "android-em-" in test["test-platform"]: 106 worker["kvm"] = True 107 108 artifacts = [ 109 # (artifact name prefix, in-image path) 110 ("public/logs", "{workdir}/workspace/logs/".format(**run)), 111 ("public/test", "{workdir}/artifacts/".format(**run)), 112 ( 113 "public/test_info", 114 "{workdir}/workspace/build/blobber_upload_dir/".format(**run), 115 ), 116 ] 117 118 installer = installer_url(taskdesc) 119 120 worker.setdefault("artifacts", []) 121 worker["artifacts"].extend([ 122 { 123 "name": prefix, 124 "path": os.path.join("{workdir}/workspace".format(**run), path), 125 "type": "directory", 126 "expires-after": get_expiration(config, "default"), 127 } 128 for (prefix, path) in artifacts 129 ]) 130 worker["artifacts"].append({ 131 "name": "public/xsession-errors.log", 132 "path": "{workdir}/.xsession-errors".format(**run), 133 "type": "file", 134 "expires-after": get_expiration(config, "default"), 135 }) 136 docker_worker_add_artifacts(config, job, taskdesc) 137 138 env = worker.setdefault("env", {}) 139 env.update({ 140 "MOZHARNESS_CONFIG": " ".join(mozharness["config"]), 141 "MOZHARNESS_SCRIPT": mozharness["script"], 142 "MOZILLA_BUILD_URL": {"artifact-reference": installer}, 143 "NEED_WINDOW_MANAGER": "true", 144 "ENABLE_E10S": str(bool(test.get("e10s"))).lower(), 145 "WORKING_DIR": "/builds/worker", 146 }) 147 148 env["PYTHON"] = "python3" 149 150 if test.get("docker-image", {}).get("in-tree") == "ubuntu1804-test": 151 env["NEED_PULSEAUDIO"] = "true" 152 153 # Bug 1602701/1601828 - use compiz on ubuntu1804 due to GTK asynchiness 154 # when manipulating windows. 155 if "wdspec" in job["run"]["test"]["suite"] or ( 156 "marionette" in job["run"]["test"]["suite"] 157 and "headless" not in job["label"] 158 ): 159 env.update({"NEED_COMPIZ": "true"}) 160 161 if test.get("docker-image", {}).get("in-tree") == "ubuntu2404-test": 162 env["NEED_PIPEWIRE"] = "true" 163 164 # Set MOZ_ENABLE_WAYLAND env variables to enable Wayland backend. 165 if "wayland" in job["label"]: 166 env["MOZ_ENABLE_WAYLAND"] = "1" 167 else: 168 assert "MOZ_ENABLE_WAYLAND" not in env 169 170 if mozharness.get("mochitest-flavor"): 171 env["MOCHITEST_FLAVOR"] = mozharness["mochitest-flavor"] 172 173 if mozharness["set-moz-node-path"]: 174 env["MOZ_NODE_PATH"] = "/usr/local/bin/node" 175 176 if "actions" in mozharness: 177 env["MOZHARNESS_ACTIONS"] = " ".join(mozharness["actions"]) 178 179 if is_try(config.params): 180 env["TRY_COMMIT_MSG"] = config.params["message"] 181 182 # handle some of the mozharness-specific options 183 if test["reboot"]: 184 raise Exception( 185 "reboot: {} not supported on generic-worker".format(test["reboot"]) 186 ) 187 188 if not test["checkout"]: 189 # Support vcs checkouts regardless of whether the task runs from 190 # source or not in case it is needed on an interactive loaner. 191 support_vcs_checkout(config, job, taskdesc, config.repo_configs) 192 193 # If we have a source checkout, run mozharness from it instead of 194 # downloading a zip file with the same content. 195 if test["checkout"]: 196 env["MOZHARNESS_PATH"] = "{workdir}/checkouts/gecko/testing/mozharness".format( 197 **run 198 ) 199 else: 200 mozharness_url = f"<build/{get_artifact_path(taskdesc, 'mozharness.zip')}>" 201 env["MOZHARNESS_URL"] = {"artifact-reference": mozharness_url} 202 203 extra_config = { 204 "installer_url": installer, 205 "test_packages_url": test_packages_url(taskdesc), 206 } 207 env["EXTRA_MOZHARNESS_CONFIG"] = { 208 "artifact-reference": json.dumps(extra_config, sort_keys=True) 209 } 210 211 # Bug 1634554 - pass in decision task artifact URL to mozharness for WPT. 212 # Bug 1645974 - test-verify-wpt and test-coverage-wpt need artifact URL. 213 if "web-platform-tests" in test["suite"] or re.match( 214 "test-(coverage|verify)-wpt", test["suite"] 215 ): 216 env["TESTS_BY_MANIFEST_URL"] = { 217 "artifact-reference": "<decision/public/tests-by-manifest.json.gz>" 218 } 219 220 command = [ 221 "{workdir}/bin/test-linux.sh".format(**run), 222 ] 223 command.extend(mozharness.get("extra-options", [])) 224 225 if test.get("test-manifests"): 226 env["MOZHARNESS_TEST_PATHS"] = json.dumps( 227 {test["suite"]: test["test-manifests"]}, sort_keys=True 228 ) 229 230 test_tags = get_test_tags(config, env) 231 if test_tags: 232 env["MOZHARNESS_TEST_TAG"] = json.dumps(test_tags) 233 command.extend([f"--tag={x}" for x in test_tags]) 234 235 # TODO: remove the need for run['chunked'] 236 elif mozharness.get("chunked") or test["chunks"] > 1: 237 command.append("--total-chunk={}".format(test["chunks"])) 238 command.append("--this-chunk={}".format(test["this-chunk"])) 239 240 if test.get("timeoutfactor"): 241 command.append("--timeout-factor={}".format(test["timeoutfactor"])) 242 243 if "download-symbols" in mozharness: 244 download_symbols = mozharness["download-symbols"] 245 download_symbols = {True: "true", False: "false"}.get( 246 download_symbols, download_symbols 247 ) 248 command.append("--download-symbols=" + download_symbols) 249 250 use_caches = test.get("use-caches", ["checkout", "pip", "uv"]) 251 job["run"] = { 252 "workdir": run["workdir"], 253 "tooltool-downloads": mozharness["tooltool-downloads"], 254 "checkout": test["checkout"], 255 "command": command, 256 "use-caches": use_caches, 257 "using": "run-task", 258 } 259 configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"]) 260 261 262 @run_job_using("generic-worker", "mozharness-test", schema=mozharness_test_run_schema) 263 def mozharness_test_on_generic_worker(config, job, taskdesc): 264 test = taskdesc["run"]["test"] 265 mozharness = test["mozharness"] 266 worker = taskdesc["worker"] = job["worker"] 267 268 bitbar_script = "test-linux.sh" 269 270 is_macosx = worker["os"] == "macosx" 271 is_windows = worker["os"] == "windows" 272 is_linux = worker["os"] == "linux" or worker["os"] in [ 273 "linux-bitbar", 274 "linux-lambda", 275 ] 276 is_bitbar = worker["os"] == "linux-bitbar" 277 is_lambda = worker["os"] == "linux-lambda" 278 assert is_macosx or is_windows or is_linux 279 280 artifacts = [ 281 { 282 "name": "public/logs", 283 "path": "logs", 284 "type": "directory", 285 "expires-after": get_expiration(config, "default"), 286 } 287 ] 288 289 generic_worker_add_artifacts(config, job, taskdesc) 290 291 # jittest doesn't have blob_upload_dir 292 if test["test-name"] != "jittest": 293 artifacts.append({ 294 "name": "public/test_info", 295 "path": "build/blobber_upload_dir", 296 "type": "directory", 297 "expires-after": get_expiration(config, "default"), 298 }) 299 300 if is_bitbar or is_lambda: 301 artifacts = [ 302 { 303 "name": "public/test/", 304 "path": "artifacts/public", 305 "type": "directory", 306 "expires-after": get_expiration(config, "default"), 307 }, 308 { 309 "name": "public/logs/", 310 "path": "workspace/logs", 311 "type": "directory", 312 "expires-after": get_expiration(config, "default"), 313 }, 314 { 315 "name": "public/test_info/", 316 "path": "workspace/build/blobber_upload_dir", 317 "type": "directory", 318 "expires-after": get_expiration(config, "default"), 319 }, 320 ] 321 322 installer = installer_url(taskdesc) 323 324 worker["os-groups"] = test["os-groups"] 325 326 # run-as-administrator is a feature for workers with UAC enabled and as such should not be 327 # included in tasks on workers that have UAC disabled. Currently UAC is only enabled on 328 # gecko Windows 10 workers, however this may be subject to change. Worker type 329 # environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig 330 # See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control 331 # for more information about UAC. 332 if test.get("run-as-administrator", False): 333 if job["worker-type"].startswith("win10-64") or job["worker-type"].startswith( 334 "win11-64" 335 ): 336 worker["run-as-administrator"] = True 337 else: 338 raise Exception( 339 "run-as-administrator not supported on {}".format(job["worker-type"]) 340 ) 341 342 if test["reboot"]: 343 raise Exception( 344 "reboot: {} not supported on generic-worker".format(test["reboot"]) 345 ) 346 347 worker["max-run-time"] = test["max-run-time"] 348 worker["retry-exit-status"] = test["retry-exit-status"] 349 worker.setdefault("artifacts", []) 350 worker["artifacts"].extend(artifacts) 351 352 env = worker.setdefault("env", {}) 353 env["GECKO_HEAD_REPOSITORY"] = config.params["head_repository"] 354 env["GECKO_HEAD_REV"] = config.params["head_rev"] 355 356 # this list will get cleaned up / reduced / removed in bug 1354088 357 if is_macosx: 358 env.update({ 359 "LC_ALL": "en_US.UTF-8", 360 "LANG": "en_US.UTF-8", 361 "MOZ_NODE_PATH": "/usr/local/bin/node", 362 "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", 363 "SHELL": "/bin/bash", 364 }) 365 elif is_bitbar or is_lambda: 366 env.update({ 367 "LANG": "en_US.UTF-8", 368 "MOZHARNESS_CONFIG": " ".join(mozharness["config"]), 369 "MOZHARNESS_SCRIPT": mozharness["script"], 370 "MOZHARNESS_URL": { 371 "artifact-reference": "<build/public/build/mozharness.zip>" 372 }, 373 "MOZILLA_BUILD_URL": {"artifact-reference": installer}, 374 "NEED_XVFB": "false", 375 "XPCOM_DEBUG_BREAK": "warn", 376 "NO_FAIL_ON_TEST_ERRORS": "1", 377 "MOZ_HIDE_RESULTS_TABLE": "1", 378 "MOZ_NODE_PATH": "/usr/local/bin/node", 379 "TASKCLUSTER_WORKER_TYPE": job["worker-type"], 380 }) 381 382 extra_config = { 383 "installer_url": installer, 384 "test_packages_url": test_packages_url(taskdesc), 385 } 386 env["EXTRA_MOZHARNESS_CONFIG"] = { 387 "artifact-reference": json.dumps(extra_config, sort_keys=True) 388 } 389 390 # Bug 1634554 - pass in decision task artifact URL to mozharness for WPT. 391 # Bug 1645974 - test-verify-wpt and test-coverage-wpt need artifact URL. 392 if "web-platform-tests" in test["suite"] or re.match( 393 "test-(coverage|verify)-wpt", test["suite"] 394 ): 395 env["TESTS_BY_MANIFEST_URL"] = { 396 "artifact-reference": "<decision/public/tests-by-manifest.json.gz>" 397 } 398 399 if is_windows: 400 py_binary = "c:\\mozilla-build\\{python}\\{python}.exe".format(python="python3") 401 mh_command = [ 402 py_binary, 403 "-u", 404 "mozharness\\scripts\\" + normpath(mozharness["script"]), 405 ] 406 elif is_bitbar or is_lambda: 407 py_binary = "python3" 408 mh_command = ["bash", f"./{bitbar_script}"] 409 elif is_macosx: 410 py_binary = "/usr/local/bin/{}".format("python3") 411 mh_command = [ 412 py_binary, 413 "-u", 414 "mozharness/scripts/" + mozharness["script"], 415 ] 416 else: 417 # is_linux 418 py_binary = "/usr/bin/{}".format("python3") 419 mh_command = [ 420 # Using /usr/bin/python2.7 rather than python2.7 because 421 # /usr/local/bin/python2.7 is broken on the mac workers. 422 # See bug #1547903. 423 py_binary, 424 "-u", 425 "mozharness/scripts/" + mozharness["script"], 426 ] 427 428 env["PYTHON"] = py_binary 429 430 for mh_config in mozharness["config"]: 431 cfg_path = "mozharness/configs/" + mh_config 432 if is_windows: 433 cfg_path = normpath(cfg_path) 434 mh_command.extend(["--cfg", cfg_path]) 435 mh_command.extend(mozharness.get("extra-options", [])) 436 if mozharness.get("download-symbols"): 437 if isinstance(mozharness["download-symbols"], str): 438 mh_command.extend(["--download-symbols", mozharness["download-symbols"]]) 439 else: 440 mh_command.extend(["--download-symbols", "true"]) 441 if mozharness.get("include-blob-upload-branch"): 442 mh_command.append("--blob-upload-branch=" + config.params["project"]) 443 444 if test.get("test-manifests"): 445 env["MOZHARNESS_TEST_PATHS"] = json.dumps( 446 {test["suite"]: test["test-manifests"]}, sort_keys=True 447 ) 448 449 test_tags = get_test_tags(config, env) 450 if test_tags: 451 # do not add --tag for perf tests 452 if test["suite"] not in ["talos", "raptor"]: 453 env["MOZHARNESS_TEST_TAG"] = json.dumps(test_tags) 454 mh_command.extend([f"--tag={x}" for x in test_tags]) 455 456 # TODO: remove the need for run['chunked'] 457 elif mozharness.get("chunked") or test["chunks"] > 1: 458 mh_command.append("--total-chunk={}".format(test["chunks"])) 459 mh_command.append("--this-chunk={}".format(test["this-chunk"])) 460 461 if test.get("timeoutfactor"): 462 mh_command.append("--timeout-factor={}".format(test["timeoutfactor"])) 463 464 if is_try(config.params): 465 env["TRY_COMMIT_MSG"] = config.params["message"] 466 467 worker["mounts"] = [ 468 { 469 "directory": "mozharness", 470 "content": { 471 "artifact": get_artifact_path(taskdesc, "mozharness.zip"), 472 "task-id": {"task-reference": "<build>"}, 473 }, 474 "format": "zip", 475 } 476 ] 477 if is_bitbar or is_lambda: 478 a_url = config.params.file_url( 479 f"taskcluster/scripts/tester/{bitbar_script}", 480 ) 481 worker["mounts"] = [ 482 { 483 "file": bitbar_script, 484 "content": { 485 "url": a_url, 486 }, 487 } 488 ] 489 490 use_caches = test.get("use-caches", ["checkout", "pip", "uv"]) 491 job["run"] = { 492 "tooltool-downloads": mozharness["tooltool-downloads"], 493 "checkout": test["checkout"], 494 "command": mh_command, 495 "use-caches": use_caches, 496 "using": "run-task", 497 } 498 if is_bitbar or is_lambda: 499 job["run"]["run-as-root"] = True 500 configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"])