beetmover_apt.py (4259B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 from itertools import islice 5 6 from taskgraph import MAX_DEPENDENCIES 7 from taskgraph.transforms.base import TransformSequence 8 from taskgraph.util.dependencies import get_primary_dependency 9 10 from gecko_taskgraph.util.platforms import architecture 11 from gecko_taskgraph.util.scriptworker import ( 12 generate_artifact_registry_gcs_sources, 13 get_beetmover_apt_repo_scope, 14 get_beetmover_repo_action_scope, 15 ) 16 17 transforms = TransformSequence() 18 19 20 @transforms.add 21 def beetmover_apt(config, tasks): 22 product = ( 23 "firefox" 24 if config.params["release_type"] == "nightly" 25 else config.params["release_product"] 26 ) 27 filtered_tasks = filter_beetmover_apt_tasks(config, tasks, product) 28 # There are too many beetmover-repackage dependencies for a single task 29 # and we hit the taskgraph dependencies limit. 30 # To work around this limitation, we chunk the would be task 31 # into tasks dependendent on, at most, half of MAX_DEPENDENCIES. 32 batches = batched(filtered_tasks, MAX_DEPENDENCIES // 2) 33 for index, batch in enumerate(batches): 34 dependencies = {} 35 gcs_sources = [] 36 for task in batch: 37 dep = get_primary_dependency(config, task) 38 assert dep 39 40 dependencies[dep.label] = dep.label 41 gcs_sources.extend(generate_artifact_registry_gcs_sources(dep)) 42 description = f"Batch {index + 1} of beetmover APT submissions for the {config.params['release_type']} .deb packages" 43 platform = f"{product}-release/opt" 44 treeherder = { 45 "platform": platform, 46 "tier": 1, 47 "kind": "other", 48 "symbol": f"BM-apt(batch-{index + 1})", 49 } 50 apt_repo_scope = get_beetmover_apt_repo_scope(config) 51 repo_action_scope = get_beetmover_repo_action_scope(config) 52 attributes = { 53 "required_signoffs": ["mar-signing"], 54 "shippable": True, 55 "shipping_product": product, 56 } 57 task = { 58 "label": f"{config.kind}-{index + 1}-{platform}", 59 "description": description, 60 "worker-type": "beetmover", 61 "treeherder": treeherder, 62 "scopes": [apt_repo_scope, repo_action_scope], 63 "attributes": attributes, 64 "shipping-phase": "ship", 65 "shipping-product": product, 66 "dependencies": dependencies, 67 } 68 worker = { 69 "implementation": "beetmover-import-from-gcs-to-artifact-registry", 70 "product": product, 71 "gcs-sources": gcs_sources, 72 } 73 task["worker"] = worker 74 yield task 75 76 77 def batched(iterable, n): 78 "Batch data into tuples of length n. The last batch may be shorter." 79 # batched('ABCDEFG', 3) --> ABC DEF G 80 if n < 1: 81 raise ValueError("n must be at least one") 82 it = iter(iterable) 83 batch = tuple(islice(it, n)) 84 while batch: 85 yield batch 86 batch = tuple(islice(it, n)) 87 88 89 def filter_beetmover_apt_tasks(config, tasks, product): 90 for task in tasks: 91 task["primary-dependency"] = get_primary_dependency(config, task) 92 if filter_beetmover_apt_task(task, product): 93 yield task 94 95 96 def filter_beetmover_apt_task(task, product): 97 # We only create beetmover-apt tasks for l10n beetmover-repackage tasks that 98 # beetmove langpack .deb packages. The langpack .deb packages support all 99 # architectures, so we generate them only on x86_64 tasks. 100 return ( 101 is_x86_64_l10n_task(task) or is_not_l10n_task(task) 102 ) and is_task_for_product(task, product) 103 104 105 def is_x86_64_l10n_task(task): 106 dep = task["primary-dependency"] 107 locale = dep.attributes.get("locale") 108 return locale and architecture(dep.attributes["build_platform"]) == "x86_64" 109 110 111 def is_not_l10n_task(task): 112 dep = task["primary-dependency"] 113 locale = dep.attributes.get("locale") 114 return not locale 115 116 117 def is_task_for_product(task, product): 118 dep = task["primary-dependency"] 119 return dep.attributes.get("shipping_product") == product