source_test.py (7336B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Source-test jobs can run on multiple platforms. These transforms allow jobs 6 with either `platform` or a list of `platforms`, and set the appropriate 7 treeherder configuration and attributes for that platform. 8 """ 9 10 import copy 11 import os 12 13 from taskgraph.transforms.base import TransformSequence 14 from taskgraph.util.attributes import keymatch 15 from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by 16 from taskgraph.util.treeherder import join_symbol, split_symbol 17 from voluptuous import Any, Extra, Optional, Required 18 19 from gecko_taskgraph.transforms.job import job_description_schema 20 21 source_test_description_schema = Schema({ 22 # most fields are passed directly through as job fields, and are not 23 # repeated here 24 Extra: object, 25 # The platform on which this task runs. This will be used to set up attributes 26 # (for try selection) and treeherder metadata (for display). If given as a list, 27 # the job will be "split" into multiple tasks, one with each platform. 28 Required("platform"): Any(str, [str]), 29 # Build labels required for the task. If this key is provided it must 30 # contain a build label for the task platform. 31 # The task will then depend on a build task, and the installer url will be 32 # saved to the GECKO_INSTALLER_URL environment variable. 33 Optional("require-build"): optionally_keyed_by("project", {str: str}), 34 # These fields can be keyed by "platform", and are otherwise identical to 35 # job descriptions. 36 Required("worker-type"): optionally_keyed_by( 37 "platform", job_description_schema["worker-type"] 38 ), 39 Required("worker"): optionally_keyed_by( 40 "platform", job_description_schema["worker"] 41 ), 42 Optional("dependencies"): { 43 k: optionally_keyed_by("platform", v) 44 for k, v in job_description_schema["dependencies"].items() 45 }, 46 # A list of artifacts to install from 'fetch' tasks. 47 Optional("fetches"): { 48 str: optionally_keyed_by("platform", job_description_schema["fetches"][str]), 49 }, 50 }) 51 52 transforms = TransformSequence() 53 54 transforms.add_validate(source_test_description_schema) 55 56 57 @transforms.add 58 def set_job_name(config, jobs): 59 for job in jobs: 60 if "task-from" in job and job["task-from"] != "kind.yml": 61 from_name = os.path.splitext(job["task-from"])[0] 62 job["name"] = "{}-{}".format(from_name, job["name"]) 63 yield job 64 65 66 @transforms.add 67 def expand_platforms(config, jobs): 68 for job in jobs: 69 if isinstance(job["platform"], str): 70 yield job 71 continue 72 73 for platform in job["platform"]: 74 pjob = copy.deepcopy(job) 75 pjob["platform"] = platform 76 77 if "name" in pjob: 78 pjob["name"] = "{}-{}".format(pjob["name"], platform) 79 else: 80 pjob["label"] = "{}-{}".format(pjob["label"], platform) 81 yield pjob 82 83 84 @transforms.add 85 def split_jsshell(config, jobs): 86 all_shells = {"sm": "Spidermonkey", "v8": "Google V8"} 87 88 for job in jobs: 89 if not job["name"].startswith("jsshell"): 90 yield job 91 continue 92 93 test = job.pop("test") 94 for shell in job.get("shell", all_shells.keys()): 95 assert shell in all_shells 96 97 new_job = copy.deepcopy(job) 98 new_job["name"] = "{}-{}".format(new_job["name"], shell) 99 new_job["description"] = "{} on {}".format( 100 new_job["description"], all_shells[shell] 101 ) 102 new_job["shell"] = shell 103 104 group = f"js-bench-{shell}" 105 symbol = split_symbol(new_job["treeherder"]["symbol"])[1] 106 new_job["treeherder"]["symbol"] = join_symbol(group, symbol) 107 108 run = new_job["run"] 109 run["mach"] = run["mach"].format( 110 shell=shell, SHELL=shell.upper(), test=test 111 ) 112 yield new_job 113 114 115 def add_build_dependency(config, job): 116 """ 117 Add build dependency to the job and installer_url to env. 118 """ 119 key = job["platform"] 120 build_labels = job.pop("require-build", {}) 121 matches = keymatch(build_labels, key) 122 if not matches: 123 raise Exception( 124 "No build platform found. " 125 f"Define 'require-build' for {key} in the task config." 126 ) 127 128 if len(matches) > 1: 129 raise Exception(f"More than one build platform found for '{key}'.") 130 131 label = matches[0] 132 deps = job.setdefault("dependencies", {}) 133 deps.update({"build": label}) 134 135 136 @transforms.add 137 def handle_platform(config, jobs): 138 """ 139 Handle the 'platform' property, setting up treeherder context as well as 140 try-related attributes. 141 """ 142 fields = [ 143 "always-target", 144 "fetches.toolchain", 145 "require-build", 146 "worker-type", 147 "worker", 148 ] 149 150 for job in jobs: 151 platform = job["platform"] 152 153 for field in fields: 154 resolve_keyed_by( 155 job, field, item_name=job["name"], project=config.params["project"] 156 ) 157 for field in job.get("dependencies", {}): 158 resolve_keyed_by( 159 job, 160 f"dependencies.{field}", 161 item_name=job["name"], 162 project=config.params["project"], 163 ) 164 165 if "treeherder" in job: 166 job["treeherder"].setdefault("platform", platform) 167 168 if "require-build" in job: 169 add_build_dependency(config, job) 170 171 del job["platform"] 172 yield job 173 174 175 @transforms.add 176 def handle_shell(config, jobs): 177 """ 178 Handle the 'shell' property. 179 """ 180 fields = [ 181 "run-on-projects", 182 "worker.env", 183 ] 184 185 for job in jobs: 186 if not job.get("shell"): 187 yield job 188 continue 189 190 for field in fields: 191 resolve_keyed_by(job, field, item_name=job["name"]) 192 193 del job["shell"] 194 yield job 195 196 197 @transforms.add 198 def set_code_review_env(config, jobs): 199 """ 200 Add a CODE_REVIEW environment variable when running in code-review bot mode 201 """ 202 is_code_review = config.params["target_tasks_method"] == "codereview" 203 204 for job in jobs: 205 attrs = job.get("attributes", {}) 206 if is_code_review and attrs.get("code-review") is True: 207 env = job["worker"].setdefault("env", {}) 208 env["CODE_REVIEW"] = "1" 209 210 yield job 211 212 213 @transforms.add 214 def remove_optimization_on_central(config, jobs): 215 """ 216 For pushes to mozilla-central run all source-test tasks that are enabled for 217 code-review in order to have the code-review bot populate the DB according 218 with the push hash. 219 """ 220 if ( 221 config.params["project"] != "mozilla-central" 222 or config.params["tasks_for"] != "hg-push" 223 ): 224 yield from jobs 225 return 226 227 for job in jobs: 228 if not job.get("attributes", {}).get("code-review", False): 229 yield job 230 continue 231 if "when" in job: 232 del job["when"] 233 if "optimization" in job and "skip-unless-mozlint" in job["optimization"]: 234 del job["optimization"] 235 yield job