build.py (10714B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Apply some defaults and minor modifications to the jobs defined in the build 6 kind. 7 """ 8 9 import logging 10 11 from mozbuild.artifact_builds import JOB_CHOICES as ARTIFACT_JOBS 12 from taskgraph.transforms.base import TransformSequence 13 from taskgraph.util.schema import resolve_keyed_by 14 from taskgraph.util.treeherder import add_suffix 15 16 from gecko_taskgraph.util.attributes import RELEASE_PROJECTS, is_try, release_level 17 from gecko_taskgraph.util.workertypes import worker_type_implementation 18 19 logger = logging.getLogger(__name__) 20 21 transforms = TransformSequence() 22 23 24 @transforms.add 25 def set_defaults(config, jobs): 26 """Set defaults, including those that differ per worker implementation""" 27 for job in jobs: 28 job["treeherder"].setdefault("kind", "build") 29 job["treeherder"].setdefault("tier", 1) 30 _, worker_os = worker_type_implementation( 31 config.graph_config, config.params, job["worker-type"] 32 ) 33 worker = job.setdefault("worker", {}) 34 worker.setdefault("env", {}) 35 worker["chain-of-trust"] = True 36 yield job 37 38 39 @transforms.add 40 def stub_installer(config, jobs): 41 for job in jobs: 42 resolve_keyed_by( 43 job, 44 "stub-installer", 45 item_name=job["name"], 46 project=config.params["project"], 47 **{ 48 "release-type": config.params["release_type"], 49 }, 50 ) 51 job.setdefault("attributes", {}) 52 if job.get("stub-installer"): 53 job["attributes"]["stub-installer"] = job["stub-installer"] 54 job["worker"]["env"].update({"USE_STUB_INSTALLER": "1"}) 55 if "stub-installer" in job: 56 del job["stub-installer"] 57 yield job 58 59 60 @transforms.add 61 def resolve_shipping_product(config, jobs): 62 for job in jobs: 63 resolve_keyed_by( 64 job, 65 "shipping-product", 66 item_name=job["name"], 67 **{ 68 "release-type": config.params["release_type"], 69 }, 70 ) 71 yield job 72 73 74 @transforms.add 75 def update_channel(config, jobs): 76 keys = [ 77 "run.update-channel", 78 "run.mar-channel-id", 79 "run.accepted-mar-channel-ids", 80 ] 81 for job in jobs: 82 job["worker"].setdefault("env", {}) 83 for key in keys: 84 resolve_keyed_by( 85 job, 86 key, 87 item_name=job["name"], 88 **{ 89 "project": config.params["project"], 90 "release-type": config.params["release_type"], 91 }, 92 ) 93 update_channel = job["run"].pop("update-channel", None) 94 if update_channel: 95 job["run"].setdefault("extra-config", {})["update_channel"] = update_channel 96 job["attributes"]["update-channel"] = update_channel 97 mar_channel_id = job["run"].pop("mar-channel-id", None) 98 if mar_channel_id: 99 job["attributes"]["mar-channel-id"] = mar_channel_id 100 job["worker"]["env"]["MAR_CHANNEL_ID"] = mar_channel_id 101 accepted_mar_channel_ids = job["run"].pop("accepted-mar-channel-ids", None) 102 if accepted_mar_channel_ids: 103 job["attributes"]["accepted-mar-channel-ids"] = accepted_mar_channel_ids 104 job["worker"]["env"]["ACCEPTED_MAR_CHANNEL_IDS"] = accepted_mar_channel_ids 105 106 yield job 107 108 109 @transforms.add 110 def mozconfig(config, jobs): 111 for job in jobs: 112 resolve_keyed_by( 113 job, 114 "run.mozconfig-variant", 115 item_name=job["name"], 116 **{ 117 "release-type": config.params["release_type"], 118 }, 119 ) 120 mozconfig_variant = job["run"].pop("mozconfig-variant", None) 121 if mozconfig_variant: 122 job["run"].setdefault("extra-config", {})["mozconfig_variant"] = ( 123 mozconfig_variant 124 ) 125 yield job 126 127 128 @transforms.add 129 def use_artifact(config, jobs): 130 if is_try(config.params): 131 use_artifact = config.params["try_task_config"].get( 132 "use-artifact-builds", False 133 ) 134 else: 135 use_artifact = False 136 for job in jobs: 137 if ( 138 config.kind == "build" 139 and use_artifact 140 and job.get("index", {}).get("job-name") in ARTIFACT_JOBS 141 # If tests aren't packaged, then we are not able to rebuild all the packages 142 and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS") == "1" 143 # Android shippable artifact builds are not supported 144 and not ( 145 "android" in job["name"] and job["attributes"].get("shippable", False) 146 ) 147 ): 148 job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"], "a") 149 job["worker"]["env"]["USE_ARTIFACT"] = "1" 150 job["attributes"]["artifact-build"] = True 151 yield job 152 153 154 @transforms.add 155 def use_profile_data(config, jobs): 156 for job in jobs: 157 use_pgo = job.pop("use-pgo", False) 158 disable_pgo = config.params["try_task_config"].get("disable-pgo", False) 159 artifact_build = job["attributes"].get("artifact-build") 160 if not use_pgo or disable_pgo or artifact_build: 161 yield job 162 continue 163 164 # If use_pgo is True, the task uses the generate-profile task of the 165 # same name. Otherwise a task can specify a specific generate-profile 166 # task to use in the use_pgo field. 167 if use_pgo is True: 168 name = job["name"] 169 else: 170 name = use_pgo 171 dependencies = f"generate-profile-{name}" 172 job.setdefault("dependencies", {})["generate-profile"] = dependencies 173 job.setdefault("fetches", {})["generate-profile"] = ["profdata.tar.xz"] 174 job["worker"]["env"].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"}) 175 176 _, worker_os = worker_type_implementation( 177 config.graph_config, config.params, job["worker-type"] 178 ) 179 if worker_os == "linux": 180 # LTO linkage needs more open files than the default from run-task. 181 job["worker"]["env"].update({"MOZ_LIMIT_NOFILE": "8192"}) 182 183 if job.get("use-sccache"): 184 raise Exception( 185 "use-sccache is incompatible with use-pgo in {}".format(job["name"]) 186 ) 187 188 yield job 189 190 191 @transforms.add 192 def resolve_keys(config, jobs): 193 for job in jobs: 194 resolve_keyed_by( 195 job, 196 "use-sccache", 197 item_name=job["name"], 198 **{"release-level": release_level(config.params)}, 199 ) 200 yield job 201 202 203 @transforms.add 204 def enable_full_crashsymbols(config, jobs): 205 """Enable full crashsymbols on jobs with 206 'enable-full-crashsymbols' set to True and on release branches, or 207 on try""" 208 branches = RELEASE_PROJECTS | {"toolchains", "try", "try-comm-central"} 209 for job in jobs: 210 enable_full_crashsymbols = job["attributes"].get("enable-full-crashsymbols") 211 if enable_full_crashsymbols and config.params["project"] in branches: 212 logger.debug("Enabling full symbol generation for %s", job["name"]) 213 job["worker"]["env"]["MOZ_ENABLE_FULL_SYMBOLS"] = "1" 214 else: 215 logger.debug("Disabling full symbol generation for %s", job["name"]) 216 job["attributes"].pop("enable-full-crashsymbols", None) 217 yield job 218 219 220 @transforms.add 221 def set_expiry(config, jobs): 222 for job in jobs: 223 attributes = job["attributes"] 224 if ( 225 "shippable" in attributes 226 and attributes["shippable"] 227 and config.kind 228 in { 229 "build", 230 } 231 ): 232 expiration_policy = "long" 233 else: 234 expiration_policy = "medium" 235 236 job["expiration-policy"] = expiration_policy 237 yield job 238 239 240 @transforms.add 241 def add_signing_artifacts(config, jobs): 242 """ 243 Add signing artifacts to macOS build jobs. 244 """ 245 is_prod_project = release_level(config.params) == "production" 246 for job in jobs: 247 if "macosx" not in job["name"] or "searchfox" in job["name"]: 248 # Not macosx build or no artifacts defined, so skip 249 yield job 250 continue 251 assert "artifacts" in job["worker"], ( 252 "macosx build jobs must have worker.artifacts defined." 253 ) 254 is_shippable = ( 255 ("shippable" in job["attributes"] and job["attributes"]["shippable"]) 256 # Instrumented builds don't have attributes.shippable set 257 or "shippable" in job["name"] 258 ) 259 260 entitlement_directory = "developer" 261 if is_shippable and is_prod_project: 262 entitlement_directory = "production" 263 264 # Decide which browser entitlement to use 265 if entitlement_directory == "developer": 266 # Try/debug builds 267 browser_entitlement = "browser" 268 elif "devedition" in job.get("shipping-product", ""): 269 # Devedition 270 browser_entitlement = "firefoxdeveloperedition.browser" 271 elif "mozilla-central" == config.params["project"]: 272 # Nightly 273 browser_entitlement = "nightly.browser" 274 else: 275 # Release and Beta 276 browser_entitlement = "firefox.browser" 277 278 for entry in job.get("worker", {}).get("artifacts", []): 279 for key in ("name", "path"): 280 if key in entry: 281 entry[key] = entry[key].format( 282 entitlement_directory=entitlement_directory, 283 browser_entitlement=browser_entitlement, 284 ) 285 # Add utility.xml if not prod/shippable 286 if not is_prod_project or not is_shippable: 287 job["worker"]["artifacts"].append({ 288 "name": "public/build/security/utility.xml", 289 "path": "checkouts/gecko/security/mac/hardenedruntime/developer/utility.xml", 290 "type": "file", 291 }) 292 impl, _ = worker_type_implementation( 293 config.graph_config, config.params, job["worker-type"] 294 ) 295 if impl == "docker-worker": 296 # For builds using docker-worker we can't use relative paths 297 # Once we switch builds to generic-worker, this can be removed 298 for entry in job.get("worker", {}).get("artifacts", []): 299 if entry.get("path", "").startswith("checkouts/gecko/security"): 300 entry["path"] = "/builds/worker/" + entry["path"] 301 yield job