diffoscope.py (6321B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 This transform construct tasks to perform diffs between builds, as 6 defined in kind.yml 7 """ 8 9 from taskgraph.transforms.base import TransformSequence 10 from taskgraph.util.schema import Schema 11 from taskgraph.util.taskcluster import get_artifact_path 12 from voluptuous import Any, Optional, Required 13 14 from gecko_taskgraph.transforms.task import task_description_schema 15 16 index_or_string = Any( 17 str, 18 {Required("index-search"): str}, 19 ) 20 21 diff_description_schema = Schema({ 22 # Name of the diff task. 23 Required("name"): str, 24 # Treeherder tier. 25 Required("tier"): int, 26 # Treeherder symbol. 27 Required("symbol"): str, 28 # relative path (from config.path) to the file the task was defined in. 29 Optional("task-from"): str, 30 # Original and new builds to compare. 31 Required("original"): index_or_string, 32 Required("new"): index_or_string, 33 # Arguments to pass to diffoscope, used for task-defaults in 34 # taskcluster/kinds/diffoscope/kind.yml 35 Optional("args"): str, 36 # Extra arguments to pass to diffoscope, that can be set per job. 37 Optional("extra-args"): str, 38 # Fail the task when differences are detected. 39 Optional("fail-on-diff"): bool, 40 # What artifact to check the differences of. Defaults to target.tar.xz 41 # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for 42 # Android. 43 Optional("artifact"): str, 44 # Whether to unpack first. Diffoscope can normally work without unpacking, 45 # but when one needs to --exclude some contents, that doesn't work out well 46 # if said content is packed (e.g. in omni.ja). 47 Optional("unpack"): bool, 48 # Commands to run before performing the diff. 49 Optional("pre-diff-commands"): [str], 50 # Only run the task on a set of projects/branches. 51 Optional("run-on-projects"): task_description_schema["run-on-projects"], 52 Optional("run-on-repo-type"): task_description_schema["run-on-repo-type"], 53 Optional("optimization"): task_description_schema["optimization"], 54 }) 55 56 transforms = TransformSequence() 57 transforms.add_validate(diff_description_schema) 58 59 60 @transforms.add 61 def fill_template(config, tasks): 62 dummy_tasks = {} 63 64 for task in tasks: 65 name = task["name"] 66 67 deps = {} 68 urls = {} 69 previous_artifact = None 70 artifact = task.get("artifact") 71 for k in ("original", "new"): 72 value = task[k] 73 if isinstance(value, str): 74 deps[k] = value 75 dep_name = k 76 os_hint = value 77 else: 78 index = value["index-search"] 79 if index not in dummy_tasks: 80 dummy_tasks[index] = { 81 "label": "index-search-" + index, 82 "description": index, 83 "worker-type": "invalid/always-optimized", 84 "run": { 85 "using": "always-optimized", 86 }, 87 "optimization": { 88 "index-search": [index], 89 }, 90 } 91 yield dummy_tasks[index] 92 deps[index] = "index-search-" + index 93 dep_name = index 94 os_hint = index.split(".")[-1] 95 if artifact: 96 pass 97 elif "linux" in os_hint: 98 artifact = "target.tar.xz" 99 elif "macosx" in os_hint: 100 artifact = "target.dmg" 101 elif "android" in os_hint: 102 artifact = "target.apk" 103 elif "win" in os_hint: 104 artifact = "target.zip" 105 else: 106 raise Exception(f"Cannot figure out the OS for {value!r}") 107 if previous_artifact is not None and previous_artifact != artifact: 108 raise Exception("Cannot compare builds from different OSes") 109 urls[k] = { 110 "artifact-reference": f"<{dep_name}/{get_artifact_path(task, artifact)}>", 111 } 112 previous_artifact = artifact 113 114 taskdesc = { 115 "label": "diff-" + name, 116 "description": name, 117 "treeherder": { 118 "symbol": task["symbol"], 119 "platform": "diff/opt", 120 "kind": "other", 121 "tier": task["tier"], 122 }, 123 "worker-type": "b-linux", 124 "worker": { 125 "docker-image": {"in-tree": "diffoscope"}, 126 "artifacts": [ 127 { 128 "type": "file", 129 "path": f"/builds/worker/{f}", 130 "name": f"public/{f}", 131 } 132 for f in ( 133 "diff.html", 134 "diff.txt", 135 ) 136 ], 137 "env": { 138 "ORIG_URL": urls["original"], 139 "NEW_URL": urls["new"], 140 "DIFFOSCOPE_ARGS": " ".join( 141 task[k] for k in ("args", "extra-args") if k in task 142 ), 143 "PRE_DIFF": "; ".join(task.get("pre-diff-commands", [])), 144 }, 145 "max-run-time": 1800, 146 }, 147 "run": { 148 "using": "run-task", 149 "checkout": task.get("unpack", False), 150 "command": "/builds/worker/bin/get_and_diffoscope{}{}".format( 151 " --unpack" if task.get("unpack") else "", 152 " --fail" if task.get("fail-on-diff") else "", 153 ), 154 }, 155 "dependencies": deps, 156 "optimization": task.get("optimization"), 157 "run-on-repo-type": task.get("run-on-repo-type", ["git", "hg"]), 158 } 159 if "run-on-projects" in task: 160 taskdesc["run-on-projects"] = task["run-on-projects"] 161 162 if artifact.endswith(".dmg"): 163 taskdesc.setdefault("fetches", {}).setdefault("toolchain", []).extend([ 164 "linux64-cctools-port", 165 "linux64-libdmg", 166 ]) 167 168 yield taskdesc