reverse_chunk_deps.py (1375B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 """ 5 Adjust dependencies to not exceed MAX_DEPENDENCIES 6 """ 7 8 from taskgraph import MAX_DEPENDENCIES 9 from taskgraph.transforms.base import TransformSequence 10 from taskgraph.util.copy import deepcopy 11 from taskgraph.util.treeherder import add_suffix 12 13 from gecko_taskgraph.transforms import release_deps 14 15 transforms = TransformSequence() 16 17 18 def yield_job(orig_job, deps, count): 19 job = deepcopy(orig_job) 20 job["dependencies"] = deps 21 job["name"] = "{}-{}".format(orig_job["name"], count) 22 if "treeherder" in job: 23 job["treeherder"]["symbol"] = add_suffix( 24 job["treeherder"]["symbol"], f"-{count}" 25 ) 26 27 return job 28 29 30 @transforms.add 31 def add_dependencies(config, jobs): 32 for job in release_deps.add_dependencies(config, jobs): 33 count = 1 34 deps = {} 35 36 # sort for deterministic chunking 37 for dep_label in sorted(job["dependencies"].keys()): 38 deps[dep_label] = dep_label 39 if len(deps) == MAX_DEPENDENCIES: 40 yield yield_job(job, deps, count) 41 deps = {} 42 count += 1 43 if deps: 44 yield yield_job(job, deps, count) 45 count += 1