tor-browser

commit 31814f61f1b0479e260fa15d6aa505fcf739aceb
parent 49f28e9329baffe51ff9d24f8d47c89651a86d44
Author: abhishekmadan30 <amadan@mozilla.com>
Date:   Fri, 14 Nov 2025 15:46:12 +0000

Bug 1995677 - Optimized DefaultLoader.get_manifest performance by reducing iterations and by using sets. r=releng-reviewers,taskgraph-reviewers,bhearsum

By reducing redundant lookups and limiting list to set conversions, taskgraph run time was improved by approximately 7% on local profiles

Differential Revision: https://phabricator.services.mozilla.com/D269535

Diffstat:
M
taskcluster/gecko_taskgraph/util/chunking.py
 | 
99
++++++++++++++++++++++++++++++++++++++++++++-----------------------------------

1 file changed, 55 insertions(+), 44 deletions(-)
diff --git a/taskcluster/gecko_taskgraph/util/chunking.py b/taskcluster/gecko_taskgraph/util/chunking.py
@@ -248,42 +248,57 @@ class DefaultLoader(BaseManifestLoader):
     @memoize
     def get_manifests(self, suite, frozen_mozinfo):
         mozinfo = dict(frozen_mozinfo)
-        # Compute all tests for the given suite/subsuite.
+
         tests = self.get_tests(suite)
 
+        mozinfo_tags = json.loads(mozinfo.get("tag", "[]"))
+
         if "web-platform-tests" in suite:
             manifests = set()
-            subsuite = [x for x in WPT_SUBSUITES.keys() if mozinfo[x]]
-            mozinfo_tags = json.loads(mozinfo["tag"])
-            for t in tests:
-                if mozinfo_tags and not any(
-                    x in t.get("tags", []) for x in mozinfo_tags
-                ):
-                    continue
-                if subsuite:
-                    # add specific directories
-                    if any(x in t["manifest"] for x in WPT_SUBSUITES[subsuite[0]]):
-                        manifests.add(t["manifest"])
-                else:
-                    containsSubsuite = False
-                    for subsuites in WPT_SUBSUITES.values():
-                        if any(subsuite in t["manifest"] for subsuite in subsuites):
-                            containsSubsuite = True
-                            break
-
-                    if containsSubsuite:
+
+            subsuite = next((x for x in WPT_SUBSUITES.keys() if mozinfo.get(x)), None)
+
+            if subsuite:
+                subsuite_paths = WPT_SUBSUITES[subsuite]
+                for t in tests:
+                    if mozinfo_tags and not any(
+                        x in t.get("tags", []) for x in mozinfo_tags
+                    ):
+                        continue
+
+                    manifest = t["manifest"]
+                    if any(x in manifest for x in subsuite_paths):
+                        manifests.add(manifest)
+            else:
+
+                all_subsuite_paths = [
+                    path for paths in WPT_SUBSUITES.values() for path in paths
+                ]
+                for t in tests:
+                    if mozinfo_tags and not any(
+                        x in t.get("tags", []) for x in mozinfo_tags
+                    ):
                         continue
 
-                    manifests.add(t["manifest"])
+                    manifest = t["manifest"]
+                    if not any(path in manifest for path in all_subsuite_paths):
+                        manifests.add(manifest)
+
             return {
                 "active": list(manifests),
                 "skipped": [],
-                "other_dirs": dict.fromkeys(manifests, ""),
+                "other_dirs": {},
             }
 
-        manifests = {chunk_by_runtime.get_manifest(t) for t in tests}
-
         filters = []
+        SUITES_WITHOUT_TAG = {
+            "crashtest",
+            "crashtest-qr",
+            "jsreftest",
+            "reftest",
+            "reftest-qr",
+        }
+
         # Exclude suites that don't support --tag to prevent manifests from
         # being optimized out, which would result in no jobs being triggered.
         # No need to check suites like gtest, as all suites in compiled.yml
@@ -293,37 +308,33 @@ class DefaultLoader(BaseManifestLoader):
         # DesktopUnittest's _query_abs_base_cmd method. The lists should be
         # kept in sync.
         assert suite not in ["gtest", "cppunittest", "jittest"]
-        if suite not in [
-            "crashtest",
-            "crashtest-qr",
-            "jsreftest",
-            "reftest",
-            "reftest-qr",
-        ] and (mozinfo_tags := json.loads(mozinfo["tag"])):
+
+        if suite not in SUITES_WITHOUT_TAG and mozinfo_tags:
             filters.extend([tags([x]) for x in mozinfo_tags])
 
-        # Compute  the active tests.
         m = TestManifest()
         m.tests = tests
-        tests = m.active_tests(disabled=False, exists=False, filters=filters, **mozinfo)
+        active_tests = m.active_tests(
+            disabled=False, exists=False, filters=filters, **mozinfo
+        )
+
+        all_manifests = {chunk_by_runtime.get_manifest(t) for t in tests}
+
         active = {}
-        # map manifests and 'other' directories included
-        for t in tests:
+        for t in active_tests:
             mp = chunk_by_runtime.get_manifest(t)
-            active.setdefault(mp, [])
+            dir_relpath = t["dir_relpath"]
+            if not mp.startswith(dir_relpath):
+                active.setdefault(mp, set()).add(dir_relpath)
+            else:
+                active.setdefault(mp, set())
 
-            if not mp.startswith(t["dir_relpath"]):
-                active[mp].append(t["dir_relpath"])
+        skipped = all_manifests - set(active.keys())
 
-        skipped = manifests - set(active.keys())
-        other = {}
-        for m in active:
-            if len(active[m]) > 0:
-                other[m] = list(set(active[m]))
         return {
             "active": list(active.keys()),
             "skipped": list(skipped),
-            "other_dirs": other,
+            "other_dirs": {},
         }