tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

cached_tasks.py (3075B)

      1 # This Source Code Form is subject to the terms of the Mozilla Public
      2 # License, v. 2.0. If a copy of the MPL was not distributed with this
      3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
      4 
      5 
      6 from collections import deque
      7 
      8 import taskgraph
      9 from taskgraph.transforms.base import TransformSequence
     10 
     11 from gecko_taskgraph.util.cached_tasks import add_optimization
     12 
     13 transforms = TransformSequence()
     14 
     15 
     16 def order_tasks(config, tasks):
     17    """Iterate image tasks in an order where parent tasks come first."""
     18    kind_prefix = config.kind + "-"
     19 
     20    pending = deque(tasks)
     21    task_labels = {task["label"] for task in pending}
     22    emitted = set()
     23    while pending:
     24        task = pending.popleft()
     25        parents = {
     26            task
     27            for task in task.get("dependencies", {}).values()
     28            if task in task_labels
     29            if task.startswith(kind_prefix)
     30        }
     31        if parents and not emitted.issuperset(parents):
     32            pending.append(task)
     33            continue
     34        emitted.add(task["label"])
     35        yield task
     36 
     37 
     38 def format_task_digest(cached_task):
     39    return "/".join([
     40        cached_task["type"],
     41        cached_task["name"],
     42        cached_task["digest"],
     43    ])
     44 
     45 
     46 @transforms.add
     47 def cache_task(config, tasks):
     48    if taskgraph.fast:
     49        for task in tasks:
     50            yield task
     51        return
     52 
     53    digests = {}
     54    for task in config.kind_dependencies_tasks.values():
     55        if (
     56            "cached_task" in task.attributes
     57            and task.attributes["cached_task"] is not False
     58        ):
     59            digests[task.label] = format_task_digest(task.attributes["cached_task"])
     60 
     61    for task in order_tasks(config, tasks):
     62        cache = task.pop("cache", None)
     63        if cache is None:
     64            yield task
     65            continue
     66 
     67        dependency_digests = []
     68        for p in task.get("dependencies", {}).values():
     69            if p in digests:
     70                dependency_digests.append(digests[p])
     71            elif config.params["project"] == "toolchains":
     72                # The toolchains repository uses non-cached toolchain artifacts. Allow
     73                # tasks to use them.
     74                cache = None
     75                break
     76            else:
     77                raise Exception(
     78                    "Cached task {} has uncached parent task: {}".format(
     79                        task["label"], p
     80                    )
     81                )
     82 
     83        if cache is None:
     84            yield task
     85            continue
     86 
     87        digest_data = cache["digest-data"] + sorted(dependency_digests)
     88        # Ensure we don't re-use cached tasks across repo types, doing so
     89        # breaks some CoT verifications.
     90        if config.params["repository_type"] == "git":
     91            digest_data.append(config.params["repository_type"])
     92 
     93        add_optimization(
     94            config,
     95            task,
     96            cache_type=cache["type"],
     97            cache_name=cache["name"],
     98            digest_data=digest_data,
     99        )
    100        digests[task["label"]] = format_task_digest(task["attributes"]["cached_task"])
    101 
    102        yield task