release_sign_and_push_langpacks.py (5836B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the release-sign-and-push task into an actual task description. 6 """ 7 8 from taskgraph.transforms.base import TransformSequence 9 from taskgraph.util.dependencies import get_primary_dependency 10 from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by 11 from taskgraph.util.treeherder import inherit_treeherder_from_dep 12 from voluptuous import Any, Optional, Required 13 14 from gecko_taskgraph.transforms.task import task_description_schema 15 from gecko_taskgraph.util.attributes import ( 16 copy_attributes_from_dependent_job, 17 release_level, 18 ) 19 20 transforms = TransformSequence() 21 22 langpack_sign_push_description_schema = Schema({ 23 Required("label"): str, 24 Required("description"): str, 25 Required("worker-type"): optionally_keyed_by("release-level", str), 26 Required("worker"): { 27 Required("channel"): optionally_keyed_by( 28 "project", "platform", Any("listed", "unlisted") 29 ), 30 Required("upstream-artifacts"): None, # Processed here below 31 }, 32 Required("run-on-projects"): [], 33 Required("scopes"): optionally_keyed_by("release-level", [str]), 34 Required("shipping-phase"): task_description_schema["shipping-phase"], 35 Optional("task-from"): task_description_schema["task-from"], 36 Optional("attributes"): task_description_schema["attributes"], 37 Optional("dependencies"): task_description_schema["dependencies"], 38 Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], 39 }) 40 41 42 @transforms.add 43 def set_label(config, jobs): 44 for job in jobs: 45 dep_job = get_primary_dependency(config, job) 46 assert dep_job 47 48 job["label"] = f"push-langpacks-{dep_job.label}" 49 50 if "name" in job: 51 del job["name"] 52 53 yield job 54 55 56 transforms.add_validate(langpack_sign_push_description_schema) 57 58 59 @transforms.add 60 def resolve_keys(config, jobs): 61 for job in jobs: 62 dep_job = get_primary_dependency(config, job) 63 assert dep_job 64 65 resolve_keyed_by( 66 job, 67 "worker-type", 68 item_name=job["label"], 69 **{"release-level": release_level(config.params)}, 70 ) 71 resolve_keyed_by( 72 job, 73 "scopes", 74 item_name=job["label"], 75 **{"release-level": release_level(config.params)}, 76 ) 77 resolve_keyed_by( 78 job, 79 "worker.channel", 80 item_name=job["label"], 81 project=config.params["project"], 82 platform=dep_job.attributes["build_platform"], 83 ) 84 85 yield job 86 87 88 @transforms.add 89 def copy_attributes(config, jobs): 90 for job in jobs: 91 dep_job = get_primary_dependency(config, job) 92 assert dep_job 93 94 job.setdefault("attributes", {}).update( 95 copy_attributes_from_dependent_job(dep_job) 96 ) 97 job["attributes"]["chunk_locales"] = dep_job.attributes.get( 98 "chunk_locales", ["en-US"] 99 ) 100 101 yield job 102 103 104 @transforms.add 105 def filter_out_macos_jobs_but_mac_only_locales(config, jobs): 106 for job in jobs: 107 dep_job = get_primary_dependency(config, job) 108 assert dep_job 109 110 build_platform = dep_job.attributes.get("build_platform") 111 112 if build_platform in ("linux64-devedition", "linux64-shippable"): 113 yield job 114 elif ( 115 build_platform in ("macosx64-devedition", "macosx64-shippable") 116 and "ja-JP-mac" in job["attributes"]["chunk_locales"] 117 ): 118 # Other locales of the same job shouldn't be processed 119 job["attributes"]["chunk_locales"] = ["ja-JP-mac"] 120 job["label"] = job["label"].replace( 121 # Guard against a chunk 10 or chunk 1 (latter on try) weird munging 122 "-{}/".format(job["attributes"]["l10n_chunk"]), 123 "-ja-JP-mac/", 124 ) 125 yield job 126 127 128 @transforms.add 129 def make_task_description(config, jobs): 130 for job in jobs: 131 dep_job = get_primary_dependency(config, job) 132 assert dep_job 133 134 treeherder = inherit_treeherder_from_dep(job, dep_job) 135 treeherder.setdefault( 136 "symbol", "langpack(SnP{})".format(job["attributes"].get("l10n_chunk", "")) 137 ) 138 139 job["description"] = job["description"].format( 140 locales="/".join(job["attributes"]["chunk_locales"]), 141 ) 142 143 job["dependencies"] = {dep_job.kind: dep_job.label} 144 job["treeherder"] = treeherder 145 146 yield job 147 148 149 def generate_upstream_artifacts(upstream_task_ref, locales): 150 return [ 151 { 152 "taskId": {"task-reference": upstream_task_ref}, 153 "taskType": "build", 154 "paths": [ 155 "public/build{locale}/target.langpack.xpi".format( 156 locale="" if locale == "en-US" else "/" + locale 157 ) 158 for locale in locales 159 ], 160 } 161 ] 162 163 164 @transforms.add 165 def make_task_worker(config, jobs): 166 for job in jobs: 167 upstream_task_ref = get_upstream_task_ref( 168 job, expected_kinds=("build", "shippable-l10n") 169 ) 170 171 job["worker"]["implementation"] = "push-addons" 172 job["worker"]["upstream-artifacts"] = generate_upstream_artifacts( 173 upstream_task_ref, job["attributes"]["chunk_locales"] 174 ) 175 176 yield job 177 178 179 def get_upstream_task_ref(job, expected_kinds): 180 upstream_tasks = [ 181 job_kind 182 for job_kind in job["dependencies"].keys() 183 if job_kind in expected_kinds 184 ] 185 186 if len(upstream_tasks) > 1: 187 raise Exception("Only one dependency expected") 188 189 return f"<{upstream_tasks[0]}>"