partials_zucchini.py (3737B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the partials task into an actual task description. 6 """ 7 8 from taskgraph.transforms.base import TransformSequence 9 from taskgraph.util.dependencies import get_primary_dependency 10 from taskgraph.util.treeherder import inherit_treeherder_from_dep 11 12 from gecko_taskgraph.util.partials import get_builds 13 from gecko_taskgraph.util.platforms import architecture 14 15 transforms = TransformSequence() 16 17 18 # TODO: Add cert verify 19 # def identify_desired_signing_keys(project, product): 20 # if project in ["mozilla-central", "comm-central", "larch", "pine", "maple"]: 21 # return "nightly" 22 # if project == "mozilla-beta": 23 # if product == "devedition": 24 # return "nightly" 25 # return "release" 26 # if ( 27 # project in ["mozilla-release", "comm-release", "comm-beta"] 28 # or project.startswith("mozilla-esr") 29 # or project.startswith("comm-esr") 30 # ): 31 # return "release" 32 # return "dep1" 33 34 35 @transforms.add 36 def make_task_description(config, tasks): 37 # If no balrog release history, then don't generate partials 38 if not config.params.get("release_history"): 39 return 40 for task in tasks: 41 dep_task = get_primary_dependency(config, task) 42 assert dep_task 43 44 # attributes = copy_attributes_from_dependent_job(dep_job) 45 locale = task["attributes"].get("locale") 46 47 build_locale = locale or "en-US" 48 49 build_platform = task["attributes"]["build_platform"] 50 builds = get_builds( 51 config.params["release_history"], build_platform, build_locale 52 ) 53 54 # If the list is empty there's no available history for this platform 55 # and locale combination, so we can't build any partials. 56 if not builds: 57 continue 58 59 locale_suffix = "" 60 if locale: 61 locale_suffix = f"{locale}/" 62 artifact_path = f"{locale_suffix}target.complete.mar" 63 64 # Fetches from upstream repackage task 65 task["fetches"][dep_task.kind] = [artifact_path] 66 67 extra_params = [f"--target=/builds/worker/artifacts/{locale_suffix}"] 68 69 for build in sorted(builds): 70 url = builds[build]["mar_url"] 71 extra_params.append(f"--from_url={url}") 72 73 to_mar_path = None 74 for artifact in dep_task.attributes["release_artifacts"]: 75 if artifact.endswith(".complete.mar"): 76 to_mar_path = artifact 77 break 78 79 task["worker"]["env"]["TO_MAR_TASK_ID"] = { 80 "task-reference": f"<{dep_task.kind}>" 81 } 82 extra_params.extend([ 83 f"--arch={architecture(build_platform)}", 84 f"--locale={build_locale}", 85 # This isn't a great approach to resolving the source URL of to_mar, but it's the same as 86 # It's only being used to fill manifest.json information, the actual file is downloaded via run-task 87 f"--to_mar_url=https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/$TO_MAR_TASK_ID/artifacts/{to_mar_path}", 88 ]) 89 # Add a space from the last command + list of extra_params 90 task["run"]["command"] += " " + " ".join(extra_params) 91 92 task["description"] = ( 93 f"Partials task for locale '{build_locale}' for build '{build_platform}'" 94 ) 95 96 task["treeherder"] = inherit_treeherder_from_dep(task, dep_task) 97 task["treeherder"]["symbol"] = f"pz({locale or 'N'})" 98 99 task["worker"]["env"]["MAR_CHANNEL_ID"] = task["attributes"]["mar-channel-id"] 100 yield task