build_components.py (6897B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import datetime 7 8 from mozilla_version.mobile import GeckoVersion 9 from taskgraph.transforms.base import TransformSequence 10 from taskgraph.util.schema import resolve_keyed_by 11 12 from ..build_config import get_extensions, get_path 13 14 transforms = TransformSequence() 15 16 17 @transforms.add 18 def resolve_keys(config, tasks): 19 for task in tasks: 20 for field in ( 21 "include-coverage", 22 "run-on-projects", 23 "shipping-phase", 24 "run.gradlew", 25 "treeherder.symbol", 26 "dependencies.build-fat-aar", 27 ): 28 resolve_keyed_by( 29 task, 30 field, 31 item_name=task["name"], 32 **{ 33 "build-type": task["attributes"]["build-type"], 34 "component": task["attributes"]["component"], 35 }, 36 ) 37 38 yield task 39 40 41 @transforms.add 42 def handle_update_channel(config, tasks): 43 for task in tasks: 44 build_fat_aar = config.kind_dependencies_tasks[ 45 task["dependencies"]["build-fat-aar"] 46 ] 47 if build_fat_aar.attributes.get("shippable"): 48 task["worker"].setdefault("env", {}).setdefault( 49 "MOZ_UPDATE_CHANNEL", 50 build_fat_aar.attributes.get("update-channel", "default"), 51 ) 52 yield task 53 54 55 @transforms.add 56 def handle_coverage(config, tasks): 57 for task in tasks: 58 if task.pop("include-coverage", False): 59 task["run"]["gradlew"].insert(0, "-Pcoverage") 60 yield task 61 62 63 @transforms.add 64 def interpolate_missing_values(config, tasks): 65 timestamp = _get_timestamp(config) 66 version = config.params["version"] 67 nightly_version = get_nightly_version(config, version) 68 69 for task in tasks: 70 for field in ("description", "run.gradlew", "treeherder.symbol"): 71 component = task["attributes"]["component"] 72 _deep_format( 73 task, 74 field, 75 component=component, 76 nightlyVersion=nightly_version, 77 timestamp=timestamp, 78 treeherder_group=component[:25], 79 ) 80 81 yield task 82 83 84 def _get_timestamp(config): 85 push_date_string = config.params["moz_build_date"] 86 push_date_time = datetime.datetime.strptime(push_date_string, "%Y%m%d%H%M%S") 87 return push_date_time.strftime("%Y%m%d.%H%M%S-1") 88 89 90 def _get_buildid(config): 91 return config.params.moz_build_date.strftime("%Y%m%d%H%M%S") 92 93 94 def get_nightly_version(config, version): 95 buildid = _get_buildid(config) 96 parsed_version = GeckoVersion.parse(version) 97 return f"{parsed_version.major_number}.{parsed_version.minor_number}.{buildid}" 98 99 100 def craft_path_version(version, build_type, nightly_version): 101 """Helper function to craft the correct version to bake in the artifacts full 102 path section""" 103 path_version = version 104 # XXX: for nightly releases we need to s/X.0.0/X.0.<buildid>/g in versions 105 if build_type == "nightly": 106 path_version = path_version.replace(version, nightly_version) 107 return path_version 108 109 110 def _deep_format(object, field, **format_kwargs): 111 keys = field.split(".") 112 last_key = keys[-1] 113 for key in keys[:-1]: 114 object = object[key] 115 116 one_before_last_object = object 117 object = object[last_key] 118 119 if isinstance(object, str): 120 one_before_last_object[last_key] = object.format(**format_kwargs) 121 elif isinstance(object, list): 122 one_before_last_object[last_key] = [ 123 item.format(**format_kwargs) for item in object 124 ] 125 else: 126 raise ValueError(f"Unsupported type for object: {object}") 127 128 129 @transforms.add 130 def add_artifacts(config, tasks): 131 _get_timestamp(config) 132 version = config.params["version"] 133 nightly_version = get_nightly_version(config, version) 134 135 for task in tasks: 136 artifact_template = task.pop("artifact-template", {}) 137 task["attributes"]["artifacts"] = artifacts = {} 138 139 component = task["attributes"]["component"] 140 build_artifact_definitions = task.setdefault("worker", {}).setdefault( 141 "artifacts", [] 142 ) 143 144 for key in [ 145 "tests-artifact-template", 146 "lint-artifact-template", 147 "jacoco-coverage-template", 148 ]: 149 if key in task: 150 optional_artifact_template = task.pop(key, {}) 151 build_artifact_definitions.append({ 152 "type": optional_artifact_template["type"], 153 "name": optional_artifact_template["name"], 154 "path": optional_artifact_template["path"].format( 155 component_path=get_path(component) 156 ), 157 }) 158 159 if artifact_template: 160 all_extensions = get_extensions(component) 161 artifact_file_names_per_extension = { 162 extension: "{component}-{version}{timestamp}{extension}".format( 163 component=component, 164 version=version, 165 timestamp="", 166 extension=extension, 167 ) 168 for extension in all_extensions 169 } 170 # XXX: rather than adding more complex logic above, we simply post-adjust the 171 # dictionary for `nightly` types of graphs 172 if task["attributes"]["build-type"] == "nightly": 173 for ext, path in artifact_file_names_per_extension.items(): 174 if version in path: 175 artifact_file_names_per_extension[ext] = path.replace( 176 version, nightly_version 177 ) 178 179 for ( 180 extension, 181 artifact_file_name, 182 ) in artifact_file_names_per_extension.items(): 183 artifact_full_name = artifact_template["name"].format( 184 artifact_file_name=artifact_file_name, 185 ) 186 build_artifact_definitions.append({ 187 "type": artifact_template["type"], 188 "name": artifact_full_name, 189 "path": artifact_template["path"].format( 190 component_path=get_path(component), 191 component=component, 192 version=craft_path_version( 193 version, 194 task["attributes"]["build-type"], 195 nightly_version, 196 ), 197 artifact_file_name=artifact_file_name, 198 ), 199 }) 200 201 artifacts[extension] = artifact_full_name 202 203 yield task