partner_attribution.py (7020B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Transform the partner attribution task into an actual task description. 6 """ 7 8 import logging 9 from collections import defaultdict 10 11 from taskgraph.transforms.base import TransformSequence 12 from taskgraph.util import json 13 from taskgraph.util.taskcluster import get_artifact_prefix 14 15 from gecko_taskgraph.util.partners import ( 16 apply_partner_priority, 17 build_macos_attribution_dmg_command, 18 check_if_partners_enabled, 19 generate_attribution_code, 20 get_ftp_platform, 21 get_partner_config_by_kind, 22 ) 23 24 log = logging.getLogger(__name__) 25 26 transforms = TransformSequence() 27 transforms.add(check_if_partners_enabled) 28 transforms.add(apply_partner_priority) 29 30 31 @transforms.add 32 def add_command_arguments(config, tasks): 33 enabled_partners = config.params.get("release_partners") 34 attribution_config = get_partner_config_by_kind(config, config.kind) 35 36 for task in tasks: 37 dependencies = {} 38 fetches = defaultdict(set) 39 attributions = [] 40 release_artifacts = [] 41 42 task_platforms = task.pop("platforms", []) 43 44 for partner_config in attribution_config.get("configs", []): 45 # we might only be interested in a subset of all partners, eg for a respin 46 if enabled_partners and partner_config["campaign"] not in enabled_partners: 47 continue 48 attribution_code = generate_attribution_code( 49 attribution_config["defaults"], partner_config 50 ) 51 for platform in partner_config["platforms"]: 52 if platform not in task_platforms: 53 continue 54 55 for locale in partner_config["locales"]: 56 attributed_build_config = _get_attributed_build_configuration( 57 task, partner_config, platform, locale 58 ) 59 upstream_label = attributed_build_config["upstream_label"] 60 if upstream_label not in config.kind_dependencies_tasks: 61 raise Exception( 62 f"Can't find upstream task for {platform} {locale}" 63 ) 64 upstream = config.kind_dependencies_tasks[upstream_label] 65 66 # set the dependencies to just what we need rather than all of l10n 67 dependencies.update({upstream.label: upstream.label}) 68 69 fetches[upstream_label].add(attributed_build_config["fetch_config"]) 70 71 attributions.append({ 72 "input": attributed_build_config["input_path"], 73 "output": attributed_build_config["output_path"], 74 "attribution": attribution_code, 75 }) 76 release_artifacts.append( 77 attributed_build_config["release_artifact"] 78 ) 79 80 if attributions: 81 worker = task.get("worker", {}) 82 worker["chain-of-trust"] = True 83 84 task.setdefault("dependencies", {}).update(dependencies) 85 task.setdefault("fetches", {}) 86 for upstream_label, upstream_artifacts in fetches.items(): 87 task["fetches"][upstream_label] = [ 88 { 89 "artifact": upstream_artifact, 90 "dest": f"{platform}/{locale}", 91 "extract": False, 92 "verify-hash": True, 93 } 94 for upstream_artifact, platform, locale in upstream_artifacts 95 ] 96 task.setdefault("attributes", {})["release_artifacts"] = release_artifacts 97 98 _build_attribution_config(task, task_platforms, attributions) 99 100 yield task 101 102 103 def _get_attributed_build_configuration(task, partner_config, platform, locale): 104 stage_platform = platform.replace("-shippable", "") 105 artifact_file_name = _get_artifact_file_name(platform) 106 107 output_artifact = _get_output_path( 108 get_artifact_prefix(task), partner_config, platform, locale, artifact_file_name 109 ) 110 return { 111 "fetch_config": ( 112 _get_upstream_artifact_path(artifact_file_name, locale), 113 stage_platform, 114 locale, 115 ), 116 "input_path": _get_input_path(stage_platform, locale, artifact_file_name), 117 "output_path": f"/builds/worker/artifacts/{output_artifact}", 118 "release_artifact": output_artifact, 119 "upstream_label": _get_upstream_task_label(platform, locale), 120 } 121 122 123 def _get_input_path(stage_platform, locale, artifact_file_name): 124 return f"/builds/worker/fetches/{stage_platform}/{locale}/{artifact_file_name}" 125 126 127 def _get_output_path( 128 artifact_prefix, partner_config, platform, locale, artifact_file_name 129 ): 130 return "{artifact_prefix}/{partner}/{sub_partner}/{ftp_platform}/{locale}/{artifact_file_name}".format( 131 artifact_prefix=artifact_prefix, 132 partner=partner_config["campaign"], 133 sub_partner=partner_config["content"], 134 ftp_platform=get_ftp_platform(platform), 135 locale=locale, 136 artifact_file_name=artifact_file_name, 137 ) 138 139 140 def _get_artifact_file_name(platform): 141 if platform.startswith("win32"): 142 return "target.stub-installer.exe" 143 elif platform.startswith("macos"): 144 return "target.dmg" 145 else: 146 raise NotImplementedError(f'Case for platform "{platform}" is not implemented') 147 148 149 def _get_upstream_task_label(platform, locale): 150 if platform.startswith("win"): 151 if locale == "en-US": 152 upstream_label = f"repackage-signing-{platform}/opt" 153 else: 154 upstream_label = f"repackage-signing-l10n-{locale}-{platform}/opt" 155 elif platform.startswith("macos"): 156 if locale == "en-US": 157 upstream_label = f"repackage-{platform}/opt" 158 else: 159 upstream_label = f"repackage-l10n-{locale}-{platform}/opt" 160 else: 161 raise NotImplementedError(f'Case for platform "{platform}" is not implemented') 162 163 return upstream_label 164 165 166 def _get_upstream_artifact_path(artifact_file_name, locale): 167 return artifact_file_name if locale == "en-US" else f"{locale}/{artifact_file_name}" 168 169 170 def _build_attribution_config(task, task_platforms, attributions): 171 if any(p.startswith("win") for p in task_platforms): 172 worker = task.get("worker", {}) 173 worker.setdefault("env", {})["ATTRIBUTION_CONFIG"] = json.dumps( 174 attributions, sort_keys=True 175 ) 176 elif any(p.startswith("macos") for p in task_platforms): 177 run = task.setdefault("run", {}) 178 run["command"] = build_macos_attribution_dmg_command( 179 "/builds/worker/fetches/dmg/dmg", attributions 180 ) 181 else: 182 raise NotImplementedError( 183 f"Case for platforms {task_platforms} is not implemented" 184 )