docker_image.py (7592B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import logging 7 import os 8 import re 9 10 import mozpack.path as mozpath 11 import taskgraph 12 from taskgraph.transforms.base import TransformSequence 13 from taskgraph.util import json 14 from taskgraph.util.docker import create_context_tar, generate_context_hash 15 from taskgraph.util.schema import Schema 16 from voluptuous import Optional, Required 17 18 from gecko_taskgraph.util.docker import ( 19 image_path, 20 ) 21 22 from .. import GECKO 23 from .task import task_description_schema 24 25 logger = logging.getLogger(__name__) 26 27 CONTEXTS_DIR = "docker-contexts" 28 29 DIGEST_RE = re.compile("^[0-9a-f]{64}$") 30 31 IMAGE_BUILDER_IMAGE = ( 32 "mozillareleases/image_builder:6.0.0" 33 "@sha256:" 34 "734c03809c83c716c1460ed3e00519d79b14d117343d3c556cbd9218a2e7f094" 35 ) 36 37 transforms = TransformSequence() 38 39 docker_image_schema = Schema({ 40 # Name of the docker image. 41 Required("name"): str, 42 # Name of the parent docker image. 43 Optional("parent"): str, 44 # Treeherder symbol. 45 Required("symbol"): str, 46 # relative path (from config.path) to the file the docker image was defined 47 # in. 48 Optional("task-from"): str, 49 # Arguments to use for the Dockerfile. 50 Optional("args"): {str: str}, 51 # Name of the docker image definition under taskcluster/docker, when 52 # different from the docker image name. 53 Optional("definition"): str, 54 # List of package tasks this docker image depends on. 55 Optional("packages"): [str], 56 Optional("arch"): str, 57 Optional( 58 "index", 59 description="information for indexing this build so its artifacts can be discovered", 60 ): task_description_schema["index"], 61 Optional( 62 "cache", 63 description="Whether this image should be cached based on inputs.", 64 ): bool, 65 Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], 66 }) 67 68 69 transforms.add_validate(docker_image_schema) 70 71 72 @transforms.add 73 def fill_template(config, tasks): 74 if not taskgraph.fast and config.write_artifacts: 75 if not os.path.isdir(CONTEXTS_DIR): 76 os.makedirs(CONTEXTS_DIR) 77 78 for task in tasks: 79 image_name = task.pop("name") 80 job_symbol = task.pop("symbol") 81 args = task.pop("args", {}) 82 packages = task.pop("packages", []) 83 parent = task.pop("parent", None) 84 85 for p in packages: 86 if f"packages-{p}" not in config.kind_dependencies_tasks: 87 raise Exception( 88 f"Missing package job for {config.kind}-{image_name}: {p}" 89 ) 90 91 if not taskgraph.fast: 92 context_path = mozpath.relpath(image_path(image_name), GECKO) 93 if config.write_artifacts: 94 context_file = os.path.join(CONTEXTS_DIR, f"{image_name}.tar.gz") 95 logger.info(f"Writing {context_file} for docker image {image_name}") 96 context_hash = create_context_tar( 97 GECKO, context_path, context_file, args 98 ) 99 else: 100 context_hash = generate_context_hash(GECKO, context_path, args) 101 else: 102 if config.write_artifacts: 103 raise Exception("Can't write artifacts if `taskgraph.fast` is set.") 104 context_hash = "0" * 40 105 digest_data = [context_hash] 106 digest_data += [json.dumps(args, sort_keys=True)] 107 108 description = f"Build the docker image {image_name} for use by dependent tasks" 109 110 args["DOCKER_IMAGE_PACKAGES"] = " ".join(f"<{p}>" for p in packages) 111 112 # Adjust the zstandard compression level based on the execution level. 113 # We use faster compression for level 1 because we care more about 114 # end-to-end times. We use slower/better compression for other levels 115 # because images are read more often and it is worth the trade-off to 116 # burn more CPU once to reduce image size. 117 zstd_level = "3" if int(config.params["level"]) == 1 else "10" 118 119 if task.get("arch", "") == "arm64": 120 worker_type = "images-aarch64" 121 else: 122 worker_type = "images" 123 124 # include some information that is useful in reconstructing this task 125 # from JSON 126 taskdesc = { 127 "label": f"{config.kind}-{image_name}", 128 "description": description, 129 "attributes": { 130 "image_name": image_name, 131 "artifact_prefix": "public", 132 }, 133 "always-target": True, 134 "expiration-policy": "long", 135 "scopes": [], 136 "treeherder": { 137 "symbol": job_symbol, 138 "platform": "taskcluster-images/opt", 139 "kind": "other", 140 "tier": 1, 141 }, 142 "run-on-projects": [], 143 "run-on-repo-type": task.get("run-on-repo-type", ["git", "hg"]), 144 "worker-type": worker_type, 145 "worker": { 146 "implementation": "docker-worker", 147 "os": "linux", 148 "artifacts": [ 149 { 150 "type": "file", 151 "path": "/workspace/out/image.tar.zst", 152 "name": "public/image.tar.zst", 153 } 154 ], 155 "env": { 156 "CONTEXT_TASK_ID": {"task-reference": "<decision>"}, 157 "CONTEXT_PATH": f"public/docker-contexts/{image_name}.tar.gz", 158 "HASH": context_hash, 159 "PROJECT": config.params["project"], 160 "IMAGE_NAME": image_name, 161 "DOCKER_IMAGE_ZSTD_LEVEL": zstd_level, 162 "DOCKER_BUILD_ARGS": {"task-reference": json.dumps(args)}, 163 "GECKO_BASE_REPOSITORY": config.params["base_repository"], 164 "GECKO_HEAD_REPOSITORY": config.params["head_repository"], 165 "GECKO_HEAD_REV": config.params["head_rev"], 166 }, 167 "chain-of-trust": True, 168 "max-run-time": 7200, 169 # FIXME: We aren't currently propagating the exit code 170 }, 171 } 172 173 worker = taskdesc["worker"] 174 175 if image_name == "image_builder": 176 worker["docker-image"] = IMAGE_BUILDER_IMAGE 177 digest_data.append(f"image-builder-image:{IMAGE_BUILDER_IMAGE}") 178 else: 179 if task.get("arch", "") == "arm64": 180 image_builder = "image_builder_arm64" 181 else: 182 image_builder = "image_builder" 183 worker["docker-image"] = {"in-tree": image_builder} 184 deps = taskdesc.setdefault("dependencies", {}) 185 deps["docker-image"] = f"{config.kind}-{image_builder}" 186 187 if packages: 188 deps = taskdesc.setdefault("dependencies", {}) 189 for p in sorted(packages): 190 deps[p] = f"packages-{p}" 191 192 if parent: 193 deps = taskdesc.setdefault("dependencies", {}) 194 deps["parent"] = f"{config.kind}-{parent}" 195 worker["env"]["PARENT_TASK_ID"] = { 196 "task-reference": "<parent>", 197 } 198 if "index" in task: 199 taskdesc["index"] = task["index"] 200 201 if task.get("cache", True) and not taskgraph.fast: 202 taskdesc["cache"] = { 203 "type": "docker-images.v2", 204 "name": image_name, 205 "digest-data": digest_data, 206 } 207 208 yield taskdesc