partials.py (6057B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the partials task into an actual task description. 6 """ 7 8 import logging 9 10 from taskgraph.transforms.base import TransformSequence 11 from taskgraph.util.dependencies import get_primary_dependency 12 from taskgraph.util.taskcluster import get_artifact_prefix 13 from taskgraph.util.treeherder import inherit_treeherder_from_dep 14 15 from gecko_taskgraph.util.attributes import ( 16 copy_attributes_from_dependent_job, 17 release_level, 18 ) 19 from gecko_taskgraph.util.partials import get_builds 20 from gecko_taskgraph.util.platforms import architecture 21 22 logger = logging.getLogger(__name__) 23 24 transforms = TransformSequence() 25 26 27 def _generate_task_output_files(job, filenames, locale=None): 28 locale_output_path = f"{locale}/" if locale else "" 29 artifact_prefix = get_artifact_prefix(job) 30 31 data = list() 32 for filename in filenames: 33 data.append({ 34 "type": "file", 35 "path": f"/home/worker/artifacts/{filename}", 36 "name": f"{artifact_prefix}/{locale_output_path}{filename}", 37 }) 38 data.append({ 39 "type": "file", 40 "path": "/home/worker/artifacts/manifest.json", 41 "name": f"{artifact_prefix}/{locale_output_path}manifest.json", 42 }) 43 return data 44 45 46 def identify_desired_signing_keys(project, product): 47 if project in [ 48 "mozilla-central", 49 "comm-central", 50 "larch", 51 "pine", 52 "maple", 53 "cypress", 54 ]: 55 return "nightly" 56 if project == "mozilla-beta": 57 if product == "devedition": 58 return "nightly" 59 return "release" 60 if ( 61 project in ["mozilla-release", "comm-release", "comm-beta"] 62 or project.startswith("mozilla-esr") 63 or project.startswith("comm-esr") 64 ): 65 return "release" 66 return "dep1" 67 68 69 @transforms.add 70 def make_task_description(config, jobs): 71 # If no balrog release history, then don't generate partials 72 if not config.params.get("release_history"): 73 return 74 for job in jobs: 75 dep_job = get_primary_dependency(config, job) 76 assert dep_job 77 78 treeherder = inherit_treeherder_from_dep(job, dep_job) 79 treeherder.setdefault("symbol", "p(N)") 80 81 label = job.get("label", f"partials-{dep_job.label}") 82 83 dependencies = {dep_job.kind: dep_job.label} 84 85 attributes = copy_attributes_from_dependent_job(dep_job) 86 locale = dep_job.attributes.get("locale") 87 if locale: 88 attributes["locale"] = locale 89 treeherder["symbol"] = f"p({locale})" 90 attributes["shipping_phase"] = job["shipping-phase"] 91 92 build_locale = locale or "en-US" 93 94 build_platform = attributes["build_platform"] 95 builds = get_builds( 96 config.params["release_history"], build_platform, build_locale 97 ) 98 99 # If the list is empty there's no available history for this platform 100 # and locale combination, so we can't build any partials. 101 if not builds: 102 continue 103 104 extra = {"funsize": {"partials": list()}} 105 update_number = 1 106 107 locale_suffix = "" 108 if locale: 109 locale_suffix = f"{locale}/" 110 artifact_path = f"<{dep_job.kind}/{get_artifact_prefix(dep_job)}/{locale_suffix}target.complete.mar>" 111 for build in sorted(builds): 112 partial_info = { 113 "locale": build_locale, 114 "from_mar": builds[build]["mar_url"], 115 "to_mar": {"artifact-reference": artifact_path}, 116 "branch": config.params["project"], 117 "update_number": update_number, 118 "dest_mar": build, 119 } 120 if "product" in builds[build]: 121 partial_info["product"] = builds[build]["product"] 122 if "previousVersion" in builds[build]: 123 partial_info["previousVersion"] = builds[build]["previousVersion"] 124 if "previousBuildNumber" in builds[build]: 125 partial_info["previousBuildNumber"] = builds[build][ 126 "previousBuildNumber" 127 ] 128 extra["funsize"]["partials"].append(partial_info) 129 update_number += 1 130 131 level = config.params["level"] 132 133 worker = { 134 "artifacts": _generate_task_output_files(dep_job, builds.keys(), locale), 135 "implementation": "docker-worker", 136 "docker-image": {"in-tree": "funsize-update-generator"}, 137 "os": "linux", 138 "max-run-time": 3600 if "asan" in dep_job.label else 1800, 139 "chain-of-trust": True, 140 "taskcluster-proxy": True, 141 "env": { 142 "SIGNING_CERT": identify_desired_signing_keys( 143 config.params["project"], config.params["release_product"] 144 ), 145 "EXTRA_PARAMS": f"--arch={architecture(build_platform)}", 146 "MAR_CHANNEL_ID": attributes["mar-channel-id"], 147 }, 148 } 149 if release_level(config.params) == "staging": 150 worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true" 151 152 task = { 153 "label": label, 154 "description": f"{dep_job.description} Partials", 155 "worker-type": "b-linux-docker-amd", 156 "dependencies": dependencies, 157 "scopes": [], 158 "attributes": attributes, 159 "run-on-projects": dep_job.attributes.get("run_on_projects"), 160 "treeherder": treeherder, 161 "extra": extra, 162 "worker": worker, 163 } 164 165 # We only want caching on linux/windows due to bug 1436977 166 if int(level) == 3 and any([ 167 build_platform.startswith(prefix) for prefix in ["linux", "win"] 168 ]): 169 task["scopes"].append( 170 "auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/" 171 ) 172 173 yield task