partner_attribution_beetmover.py (5667B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the beetmover task into an actual task description. 6 """ 7 8 from taskgraph.transforms.base import TransformSequence 9 from taskgraph.util.dependencies import get_primary_dependency 10 from taskgraph.util.schema import Schema 11 from taskgraph.util.taskcluster import get_artifact_prefix 12 from voluptuous import Optional, Required 13 14 from gecko_taskgraph.transforms.beetmover import craft_release_properties 15 from gecko_taskgraph.transforms.task import task_description_schema 16 from gecko_taskgraph.util.attributes import ( 17 copy_attributes_from_dependent_job, 18 ) 19 from gecko_taskgraph.util.partners import ( 20 apply_partner_priority, 21 ) 22 from gecko_taskgraph.util.scriptworker import ( 23 add_scope_prefix, 24 get_beetmover_bucket_scope, 25 ) 26 27 beetmover_description_schema = Schema({ 28 # from the loader: 29 Optional("task-from"): str, 30 Optional("name"): str, 31 # from the from_deps transforms: 32 Optional("attributes"): task_description_schema["attributes"], 33 Optional("dependencies"): task_description_schema["dependencies"], 34 # depname is used in taskref's to identify the taskID of the unsigned things 35 Required("depname", default="build"): str, 36 # unique label to describe this beetmover task, defaults to {dep.label}-beetmover 37 Optional("label"): str, 38 Required("partner-path"): str, 39 Optional("extra"): object, 40 Required("shipping-phase"): task_description_schema["shipping-phase"], 41 Optional("shipping-product"): task_description_schema["shipping-product"], 42 Optional("priority"): task_description_schema["priority"], 43 Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], 44 }) 45 46 transforms = TransformSequence() 47 transforms.add_validate(beetmover_description_schema) 48 transforms.add(apply_partner_priority) 49 50 51 @transforms.add 52 def populate_scopes_and_upstream_artifacts(config, jobs): 53 for job in jobs: 54 dep_job = get_primary_dependency(config, job) 55 assert dep_job 56 57 upstream_artifacts = dep_job.attributes["release_artifacts"] 58 prefix = get_artifact_prefix(dep_job) 59 artifacts = [] 60 for artifact in upstream_artifacts: 61 partner, sub_partner, platform, locale, _ = artifact.replace( 62 prefix + "/", "" 63 ).split("/", 4) 64 artifacts.append((artifact, partner, sub_partner, platform, locale)) 65 66 action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner") 67 bucket_scope = get_beetmover_bucket_scope(config) 68 repl_dict = { 69 "build_number": config.params["build_number"], 70 "release_partner_build_number": config.params[ 71 "release_partner_build_number" 72 ], 73 "version": config.params["version"], 74 "partner": "{partner}", # we'll replace these later, per artifact 75 "subpartner": "{subpartner}", 76 "platform": "{platform}", 77 "locale": "{locale}", 78 } 79 job["scopes"] = [bucket_scope, action_scope] 80 81 partner_path = job["partner-path"].format(**repl_dict) 82 job.setdefault("worker", {})["upstream-artifacts"] = ( 83 generate_upstream_artifacts(dep_job.kind, artifacts, partner_path) 84 ) 85 86 yield job 87 88 89 @transforms.add 90 def make_task_description(config, jobs): 91 for job in jobs: 92 dep_job = get_primary_dependency(config, job) 93 assert dep_job 94 95 attributes = dep_job.attributes 96 build_platform = attributes.get("build_platform") 97 if not build_platform: 98 raise Exception("Cannot find build platform!") 99 100 description = "Beetmover for partner attribution" 101 attributes = copy_attributes_from_dependent_job(dep_job) 102 103 task = { 104 "label": "{}-{}".format(config.kind, job["name"]), 105 "description": description, 106 "dependencies": {dep_job.kind: dep_job.label}, 107 "attributes": attributes, 108 "run-on-projects": dep_job.attributes.get("run_on_projects"), 109 "run-on-repo-type": job.get("run-on-repo-type", ["git", "hg"]), 110 "shipping-phase": job["shipping-phase"], 111 "shipping-product": job.get("shipping-product"), 112 "worker": job["worker"], 113 "scopes": job["scopes"], 114 } 115 # we may have reduced the priority for partner jobs, otherwise task.py will set it 116 if job.get("priority"): 117 task["priority"] = job["priority"] 118 119 yield task 120 121 122 def generate_upstream_artifacts(attribution_task_kind, artifacts, partner_path): 123 upstream_artifacts = [] 124 for artifact, partner, subpartner, platform, locale in artifacts: 125 upstream_artifacts.append({ 126 "taskId": {"task-reference": f"<{attribution_task_kind}>"}, 127 "taskType": "repackage", 128 "paths": [artifact], 129 "locale": partner_path.format( 130 partner=partner, 131 subpartner=subpartner, 132 platform=platform, 133 locale=locale, 134 ), 135 }) 136 137 if not upstream_artifacts: 138 raise Exception("Couldn't find any upstream artifacts.") 139 140 return upstream_artifacts 141 142 143 @transforms.add 144 def make_task_worker(config, jobs): 145 for job in jobs: 146 job["worker-type"] = "beetmover" 147 worker = { 148 "implementation": "beetmover", 149 "release-properties": craft_release_properties(config, job), 150 } 151 job["worker"].update(worker) 152 153 yield job