signing.py (9695B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the signing task into an actual task description. 6 """ 7 8 from taskgraph.transforms.base import TransformSequence 9 from taskgraph.util.dependencies import get_primary_dependency 10 from taskgraph.util.keyed_by import evaluate_keyed_by 11 from taskgraph.util.schema import Schema, taskref_or_string 12 from voluptuous import Optional, Required 13 14 from gecko_taskgraph.transforms.task import task_description_schema 15 from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job 16 from gecko_taskgraph.util.scriptworker import ( 17 add_scope_prefix, 18 get_signing_type_per_platform, 19 ) 20 21 transforms = TransformSequence() 22 23 signing_description_schema = Schema({ 24 # Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py 25 # because this is passed directly into the signingscript worker 26 Required("upstream-artifacts"): [ 27 { 28 # taskId of the task with the artifact 29 Required("taskId"): taskref_or_string, 30 # type of signing task (for CoT) 31 Required("taskType"): str, 32 # Paths to the artifacts to sign 33 Required("paths"): [str], 34 # Signing formats to use on each of the paths 35 Required("formats"): [str], 36 } 37 ], 38 # attributes for this task 39 Optional("attributes"): {str: object}, 40 # unique label to describe this signing task, defaults to {dep.label}-signing 41 Optional("label"): str, 42 # treeherder is allowed here to override any defaults we use for signing. See 43 # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the 44 # below transforms for defaults of various values. 45 Optional("treeherder"): task_description_schema["treeherder"], 46 # Routes specific to this task, if defined 47 Optional("routes"): [str], 48 Optional("shipping-phase"): task_description_schema["shipping-phase"], 49 Optional("shipping-product"): task_description_schema["shipping-product"], 50 Required("dependencies"): task_description_schema["dependencies"], 51 Optional("extra"): {str: object}, 52 # Max number of partner repacks per chunk 53 Optional("repacks-per-chunk"): int, 54 # Override the default priority for the project 55 Optional("priority"): task_description_schema["priority"], 56 Optional("task-from"): task_description_schema["task-from"], 57 Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], 58 }) 59 60 61 def get_locales_description(attributes, default): 62 """Returns the [list] of locales for task description usage""" 63 chunk_locales = attributes.get("chunk_locales") 64 if chunk_locales: 65 return ", ".join(chunk_locales) 66 return attributes.get("locale", default) 67 68 69 @transforms.add 70 def delete_name(config, jobs): 71 """Delete the 'name' key if it exists, we don't use it.""" 72 for job in jobs: 73 if "name" in job: 74 del job["name"] 75 yield job 76 77 78 transforms.add_validate(signing_description_schema) 79 80 81 @transforms.add 82 def add_requirements_link(config, jobs): 83 for job in jobs: 84 dep_job = get_primary_dependency(config, job) 85 assert dep_job 86 requirements_path = evaluate_keyed_by( 87 config.graph_config["mac-signing"]["mac-requirements"], 88 "mac requirements", 89 { 90 "platform": dep_job.attributes.get("build_platform"), 91 }, 92 ) 93 if requirements_path: 94 job["requirements-plist-url"] = config.params.file_url( 95 requirements_path, 96 ) 97 yield job 98 99 100 @transforms.add 101 def make_task_description(config, jobs): 102 for job in jobs: 103 dep_job = get_primary_dependency(config, job) 104 assert dep_job 105 attributes = dep_job.attributes 106 107 formats = set() 108 for artifacts in job["upstream-artifacts"]: 109 for f in artifacts["formats"]: 110 formats.add(f) # Add each format only once 111 112 is_shippable = dep_job.attributes.get("shippable", False) 113 build_platform = dep_job.attributes.get("build_platform") 114 assert build_platform 115 treeherder = None 116 if "partner" not in config.kind and "eme-free" not in config.kind: 117 treeherder = job.get("treeherder", {}) 118 119 dep_th_platform = ( 120 dep_job.task.get("extra", {}) 121 .get("treeherder", {}) 122 .get("machine", {}) 123 .get("platform", "") 124 ) 125 build_type = dep_job.attributes.get("build_type") 126 treeherder.setdefault( 127 "platform", 128 _generate_treeherder_platform( 129 dep_th_platform, build_platform, build_type 130 ), 131 ) 132 133 # ccov builds are tier 2, so they cannot have tier 1 tasks 134 # depending on them. 135 treeherder.setdefault( 136 "tier", 137 dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1), 138 ) 139 treeherder.setdefault( 140 "symbol", 141 _generate_treeherder_symbol( 142 dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol") 143 ), 144 ) 145 treeherder.setdefault("kind", "build") 146 147 label = job["label"] 148 description = ( 149 "Signing of locale(s) '{locale}' for build '" 150 "{build_platform}/{build_type}'".format( 151 locale=get_locales_description(attributes, "en-US"), 152 build_platform=build_platform, 153 build_type=attributes.get("build_type"), 154 ) 155 ) 156 157 attributes = ( 158 job["attributes"] 159 if job.get("attributes") 160 else copy_attributes_from_dependent_job(dep_job) 161 ) 162 attributes["signed"] = True 163 164 if "linux" in build_platform: 165 attributes["release_artifacts"] = ["public/build/KEY"] 166 167 if dep_job.attributes.get("chunk_locales"): 168 # Used for l10n attribute passthrough 169 attributes["chunk_locales"] = dep_job.attributes.get("chunk_locales") 170 171 signing_type = get_signing_type_per_platform( 172 build_platform, is_shippable, config 173 ) 174 worker_type_alias = "linux-signing" if is_shippable else "linux-depsigning" 175 task = { 176 "label": label, 177 "description": description, 178 "worker": { 179 "implementation": "scriptworker-signing", 180 "signing-type": signing_type, 181 "upstream-artifacts": job["upstream-artifacts"], 182 }, 183 "dependencies": job["dependencies"], 184 "attributes": attributes, 185 "run-on-projects": dep_job.attributes.get("run_on_projects"), 186 "run-on-repo-type": job.get("run-on-repo-type", ["git", "hg"]), 187 "optimization": dep_job.optimization, 188 "routes": job.get("routes", []), 189 "shipping-product": job.get("shipping-product"), 190 "shipping-phase": job.get("shipping-phase"), 191 } 192 if dep_job.kind in task["dependencies"]: 193 task["if-dependencies"] = [dep_job.kind] 194 195 # build-mac-{signing,notarization} uses signingscript instead of iscript 196 if "macosx" in build_platform and config.kind.endswith("-mac-notarization"): 197 task["worker"]["signing-type"] = "release-apple-notarization" 198 task["scopes"] = [ 199 add_scope_prefix(config, "signing:cert:release-apple-notarization") 200 ] 201 task["description"] = ( 202 "Notarization of '{}' locales for build '{}/{}'".format( 203 get_locales_description(attributes, "en-US"), 204 build_platform, 205 attributes.get("build_type"), 206 ) 207 ) 208 task["retries"] = 0 209 elif "macosx" in build_platform: 210 # iscript overrides 211 task["worker"]["implementation"] = "iscript" 212 task["worker"]["mac-behavior"] = "mac_sign_and_pkg" 213 214 worker_type_alias_map = { 215 "linux-depsigning": "mac-depsigning", 216 "linux-signing": "mac-signing", 217 } 218 assert worker_type_alias in worker_type_alias_map, ( 219 "Make sure to adjust the below worker_type_alias logic for " 220 "mac if you change the signing workerType aliases!" 221 f" ({worker_type_alias} not found in mapping)" 222 ) 223 worker_type_alias = worker_type_alias_map[worker_type_alias] 224 for attr in ("entitlements-url", "requirements-plist-url"): 225 if job.get(attr): 226 task["worker"][attr] = job[attr] 227 228 task["worker-type"] = worker_type_alias 229 if treeherder: 230 task["treeherder"] = treeherder 231 if job.get("extra"): 232 task["extra"] = job["extra"] 233 # we may have reduced the priority for partner jobs, otherwise task.py will set it 234 if job.get("priority"): 235 task["priority"] = job["priority"] 236 237 yield task 238 239 240 def _generate_treeherder_platform(dep_th_platform, build_platform, build_type): 241 if "-pgo" in build_platform: 242 actual_build_type = "pgo" 243 elif "-ccov" in build_platform: 244 actual_build_type = "ccov" 245 else: 246 actual_build_type = build_type 247 return f"{dep_th_platform}/{actual_build_type}" 248 249 250 def _generate_treeherder_symbol(build_symbol): 251 symbol = build_symbol + "s" 252 return symbol