cached_tasks.py (3188B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import hashlib 7 import time 8 9 TARGET_CACHE_INDEX = "{trust_domain}.cache.level-{level}.{type}.{name}.hash.{digest}" 10 EXTRA_CACHE_INDEXES = [ 11 "{trust_domain}.cache.level-{level}.{type}.{name}.latest", 12 "{trust_domain}.cache.level-{level}.{type}.{name}.pushdate.{build_date_long}", 13 ] 14 15 16 def add_optimization( 17 config, taskdesc, cache_type, cache_name, digest=None, digest_data=None 18 ): 19 """ 20 Allow the results of this task to be cached. This adds index routes to the 21 task so it can be looked up for future runs, and optimization hints so that 22 cached artifacts can be found. Exactly one of `digest` and `digest_data` 23 must be passed. 24 25 :param TransformConfig config: The configuration for the kind being transformed. 26 :param dict taskdesc: The description of the current task. 27 :param str cache_type: The type of task result being cached. 28 :param str cache_name: The name of the object being cached. 29 :param digest: A unique string indentifying this version of the artifacts 30 being generated. Typically this will be the hash of inputs to the task. 31 :type digest: bytes or None 32 :param digest_data: A list of bytes representing the inputs of this task. 33 They will be concatenated and hashed to create the digest for this 34 task. 35 :type digest_data: list of bytes or None 36 """ 37 cached_task = taskdesc.get("attributes", {}).get("cached_task") 38 if cached_task is False: 39 return 40 41 if (digest is None) == (digest_data is None): 42 raise Exception("Must pass exactly one of `digest` and `digest_data`.") 43 if digest is None: 44 digest = hashlib.sha256("\n".join(digest_data).encode("utf-8")).hexdigest() 45 46 subs = { 47 "trust_domain": config.graph_config["trust-domain"], 48 "type": cache_type, 49 "name": cache_name, 50 "digest": digest, 51 } 52 53 # We'll try to find a cached version of the toolchain at levels above 54 # and including the current level, starting at the highest level. 55 index_routes = [] 56 for level in reversed(range(int(config.params["level"]), 4)): 57 subs["level"] = level 58 index_routes.append(TARGET_CACHE_INDEX.format(**subs)) 59 taskdesc["optimization"] = {"index-search": index_routes} 60 61 # ... and cache at the lowest level. 62 taskdesc.setdefault("routes", []).append( 63 f"index.{TARGET_CACHE_INDEX.format(**subs)}" 64 ) 65 66 # ... and add some extra routes for humans 67 subs["build_date_long"] = time.strftime( 68 "%Y.%m.%d.%Y%m%d%H%M%S", time.gmtime(config.params["build_date"]) 69 ) 70 taskdesc["routes"].extend([ 71 f"index.{route.format(**subs)}" for route in EXTRA_CACHE_INDEXES 72 ]) 73 74 taskdesc["attributes"]["cached_task"] = { 75 "type": cache_type, 76 "name": cache_name, 77 "digest": digest, 78 } 79 80 # Allow future pushes to find this task before it completes 81 # Implementation in morphs 82 taskdesc["attributes"]["eager_indexes"] = [TARGET_CACHE_INDEX.format(**subs)]