worker.py (8979B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 from taskgraph.transforms.base import TransformSequence 6 7 # default worker types keyed by instance-size 8 LINUX_WORKER_TYPES = { 9 "large-legacy": "t-linux-docker", 10 "large": "t-linux-docker-amd", 11 "large-noscratch": "t-linux-docker-noscratch-amd", 12 "xlarge": "t-linux-docker-amd", 13 "xlarge-noscratch": "t-linux-docker-noscratch-amd", 14 "highcpu": "t-linux-docker-16c32gb-amd", 15 "default": "t-linux-docker-noscratch-amd", 16 } 17 18 # windows worker types keyed by test-platform and virtualization 19 WINDOWS_WORKER_TYPES = { 20 "windows10-64": { # source-test 21 "virtual": "t-win10-64", 22 "virtual-with-gpu": "t-win10-64-gpu-s", 23 "hardware": "win10-64-2009-hw", 24 }, 25 "windows10-64-shippable-qr": { 26 "virtual": "t-win10-64", 27 "virtual-with-gpu": "t-win10-64-gpu-s", 28 "hardware": "win10-64-2009-hw", 29 }, 30 "windows10-64-2009": { 31 "virtual": "win10-64-2009", 32 "virtual-with-gpu": "win10-64-2009-gpu", 33 "hardware": "win10-64-2009-hw", 34 }, 35 "windows10-64-2009-qr": { 36 "virtual": "win10-64-2009", 37 "virtual-with-gpu": "win10-64-2009-gpu", 38 "hardware": "win10-64-2009-hw", 39 }, 40 "windows10-64-2009-shippable-qr": { 41 "virtual": "win10-64-2009", 42 "virtual-with-gpu": "win10-64-2009-gpu", 43 "hardware": "win10-64-2009-hw", 44 }, 45 "windows11-32-24h2-mingwclang": { 46 "virtual": "win11-64-24h2", 47 "virtual-with-gpu": "win11-64-24h2-gpu", 48 }, 49 "windows11-32-24h2": { 50 "virtual": "win11-64-24h2", 51 "virtual-with-gpu": "win11-64-24h2-gpu", 52 }, 53 "windows11-32-24h2-shippable": { 54 "virtual": "win11-64-24h2", 55 "virtual-with-gpu": "win11-64-24h2-gpu", 56 }, 57 "windows11-64-24h2-hw-ref-shippable": { 58 "virtual": "win11-64-24h2-hw-ref", 59 "virtual-with-gpu": "win11-64-24h2-hw-ref", 60 "hardware": "win11-64-24h2-hw-ref", 61 }, 62 "windows11-64-24h2-hw-ref": { 63 "virtual": "win11-64-24h2-hw-ref", 64 "virtual-with-gpu": "win11-64-24h2-hw-ref", 65 "hardware": "win11-64-24h2-hw-ref", 66 }, 67 "windows11-64-24h2": { 68 "virtual": "win11-64-24h2", 69 "virtual-with-gpu": "win11-64-24h2-gpu", 70 "hardware": "win11-64-24h2-hw", 71 }, 72 "windows11-64-24h2-ccov": { 73 "virtual": "win11-64-24h2", 74 "virtual-with-gpu": "win11-64-24h2-gpu", 75 }, 76 "windows11-64-24h2-devedition": { 77 "virtual": "win11-64-24h2", 78 "virtual-with-gpu": "win11-64-24h2-gpu", 79 }, 80 "windows11-64-24h2-shippable": { 81 "virtual": "win11-64-24h2", 82 "virtual-with-gpu": "win11-64-24h2-gpu", 83 "hardware": "win11-64-24h2-hw", 84 }, 85 "windows11-64-24h2-asan": { 86 "virtual": "win11-64-24h2", 87 "large": "win11-64-24h2-large", 88 "virtual-with-gpu": "win11-64-24h2-gpu", 89 }, 90 "windows11-64-24h2-mingwclang": { 91 "virtual": "win11-64-24h2", 92 "virtual-with-gpu": "win11-64-24h2-gpu", 93 }, 94 "windows11-aarch64-24h2": { 95 "virtual": "win11-a64-24h2", 96 }, 97 } 98 99 # os x worker types keyed by test-platform 100 MACOSX_WORKER_TYPES = { 101 "macosx1015-64": "t-osx-1015-r8", 102 "macosx1470-64": "t-osx-1400-r8", 103 "macosx1400-64": "t-osx-1400-m2", 104 "macosx1500-64": "t-osx-1500-m4", 105 "macosx1500-aarch64": "t-osx-1500-m4", 106 } 107 108 transforms = TransformSequence() 109 110 111 @transforms.add 112 def set_worker_type(config, tasks): 113 """Set the worker type based on the test platform.""" 114 for task in tasks: 115 # during the taskcluster migration, this is a bit tortured, but it 116 # will get simpler eventually! 117 test_platform = task["test-platform"] 118 if task.get("worker-type", "default") != "default": 119 # This test already has its worker type defined, so just use that (yields below) 120 # Unless the value is set to "default", in that case ignore it. 121 pass 122 elif test_platform.startswith("macosx1015-64"): 123 task["worker-type"] = MACOSX_WORKER_TYPES["macosx1015-64"] 124 elif test_platform.startswith("macosx1470-64"): 125 task["worker-type"] = MACOSX_WORKER_TYPES["macosx1470-64"] 126 elif test_platform.startswith("macosx1400-64"): 127 task["worker-type"] = MACOSX_WORKER_TYPES["macosx1400-64"] 128 elif test_platform.startswith("macosx1400-aarch64"): 129 task["worker-type"] = MACOSX_WORKER_TYPES["macosx1400-aarch64"] 130 elif test_platform.startswith("macosx1500-aarch64"): 131 task["worker-type"] = MACOSX_WORKER_TYPES["macosx1500-aarch64"] 132 elif test_platform.startswith("macosx1500-64"): 133 task["worker-type"] = MACOSX_WORKER_TYPES["macosx1500-64"] 134 elif test_platform.startswith("win"): 135 # figure out what platform the job needs to run on 136 if task["virtualization"] == "hardware": 137 # some jobs like talos and reftest run on real h/w 138 if test_platform.startswith("windows11-64-24h2-hw-ref"): 139 win_worker_type_platform = WINDOWS_WORKER_TYPES[ 140 "windows11-64-24h2-hw-ref" 141 ] 142 elif test_platform.startswith("windows10-64"): 143 win_worker_type_platform = WINDOWS_WORKER_TYPES["windows10-64"] 144 elif test_platform.startswith("windows11-64-24h2"): 145 win_worker_type_platform = WINDOWS_WORKER_TYPES["windows11-64-24h2"] 146 else: 147 raise Exception(f"Unknown worker type for {test_platform}") 148 else: 149 # the other jobs run on a vm which may or may not be a win10 vm 150 win_worker_type_platform = WINDOWS_WORKER_TYPES[ 151 test_platform.split("/")[0] 152 ] 153 if task[ 154 "virtualization" 155 ] == "virtual-with-gpu" and test_platform.startswith("windows1"): 156 # add in `--requires-gpu` to the mozharness options 157 task["mozharness"]["extra-options"].append("--requires-gpu") 158 159 # now we have the right platform set the worker type accordingly 160 if task["instance-size"].startswith("large") and test_platform.startswith( 161 "windows11-" 162 ): 163 task["worker-type"] = "win11-64-24h2-large" 164 else: 165 task["worker-type"] = win_worker_type_platform[task["virtualization"]] 166 elif test_platform.startswith("android-hw-p5"): 167 if task["suite"] != "raptor": 168 task["worker-type"] = "t-bitbar-gw-unit-p5" 169 else: 170 task["worker-type"] = "t-bitbar-gw-perf-p5" 171 elif test_platform.startswith("android-hw-p6"): 172 if task["suite"] != "raptor": 173 task["worker-type"] = "t-bitbar-gw-unit-p6" 174 else: 175 task["worker-type"] = "t-bitbar-gw-perf-p6" 176 elif test_platform.startswith("android-hw-s24"): 177 if task["suite"] != "raptor": 178 task["worker-type"] = "t-bitbar-gw-unit-s24" 179 else: 180 task["worker-type"] = "t-bitbar-gw-perf-s24" 181 elif test_platform.startswith("android-hw-a55"): 182 if task["suite"] != "raptor": 183 task["worker-type"] = "t-lambda-perf-a55" 184 else: 185 task["worker-type"] = "t-bitbar-gw-perf-a55" 186 elif test_platform.startswith("android-em-"): 187 task["worker-type"] = "t-linux-kvm" 188 elif test_platform.startswith("linux") or test_platform.startswith("android"): 189 if "wayland" in test_platform: 190 if task["instance-size"].startswith("xlarge"): 191 task["worker-type"] = "t-linux-xlarge-wayland" 192 else: 193 task["worker-type"] = "t-linux-wayland" 194 elif task.get("suite", "") in ["talos", "raptor"] and not task[ 195 "build-platform" 196 ].startswith("linux64-ccov"): 197 if "browsertime-network-bench" in task.get("test-name"): 198 task["worker-type"] = "t-linux-netperf-1804" 199 else: 200 task["worker-type"] = "t-linux-talos-1804" 201 else: 202 task["worker-type"] = LINUX_WORKER_TYPES[task["instance-size"]] 203 else: 204 raise Exception(f"unknown test_platform {test_platform}") 205 206 yield task 207 208 209 @transforms.add 210 def set_wayland_env(config, tasks): 211 for task in tasks: 212 if "wayland" not in task["test-platform"]: 213 yield task 214 continue 215 216 env = task.setdefault("worker", {}).setdefault("env", {}) 217 env["MOZ_ENABLE_WAYLAND"] = "1" 218 env["WAYLAND_DISPLAY"] = "wayland-0" 219 env["NEED_GNOME_KEYRING"] = "true" 220 yield task