chunking.py (12473B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 """Utility functions to handle test chunking.""" 7 8 import logging 9 import os 10 import traceback 11 from abc import ABCMeta, abstractmethod 12 13 from manifestparser import TestManifest 14 from manifestparser.filters import chunk_by_runtime, tags 15 from mozbuild.util import memoize 16 from mozinfo.platforminfo import PlatformInfo 17 from moztest.resolve import TEST_SUITES, TestManifestLoader, TestResolver 18 from requests.exceptions import RetryError 19 from taskgraph.util import json 20 from taskgraph.util.yaml import load_yaml 21 22 from gecko_taskgraph import GECKO, TEST_CONFIGS 23 from gecko_taskgraph.util.bugbug import CT_LOW, BugbugTimeoutException, push_schedules 24 25 logger = logging.getLogger(__name__) 26 here = os.path.abspath(os.path.dirname(__file__)) 27 resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) 28 29 VARIANTS_YML = os.path.join(TEST_CONFIGS, "variants.yml") 30 TEST_VARIANTS = {} 31 if os.path.exists(VARIANTS_YML): 32 TEST_VARIANTS = load_yaml(VARIANTS_YML) 33 34 WPT_SUBSUITES = { 35 "canvas": ["html/canvas"], 36 "webgpu": ["_mozilla/webgpu"], 37 "webcodecs": ["webcodecs"], 38 "eme": ["encrypted-media"], 39 } 40 41 42 def get_test_tags(config, env): 43 tags = json.loads( 44 config.params["try_task_config"].get("env", {}).get("MOZHARNESS_TEST_TAG", "[]") 45 ) 46 tags.extend(env.get("MOZHARNESS_TEST_TAG", [])) 47 return list(set(tags)) 48 49 50 def guess_mozinfo_from_task(task, repo="", app_version="", test_tags=[]): 51 """Attempt to build a mozinfo dict from a task definition. 52 53 This won't be perfect and many values used in the manifests will be missing. But 54 it should cover most of the major ones and be "good enough" for chunking in the 55 taskgraph. 56 57 Args: 58 task (dict): A task definition. 59 60 Returns: 61 A dict that can be used as a mozinfo replacement. 62 """ 63 setting = task["test-setting"] 64 runtime_keys = setting["runtime"].keys() 65 66 platform_info = PlatformInfo(setting) 67 68 info = { 69 "debug": platform_info.debug, 70 "bits": platform_info.bits, 71 "asan": setting["build"].get("asan", False), 72 "tsan": setting["build"].get("tsan", False), 73 "ccov": setting["build"].get("ccov", False), 74 "mingwclang": setting["build"].get("mingwclang", False), 75 "nightly_build": "a1" 76 in app_version, # https://searchfox.org/mozilla-central/source/build/moz.configure/init.configure#1101 77 "release_or_beta": "a" not in app_version, 78 "repo": repo, 79 } 80 # the following are used to evaluate reftest skip-if 81 info["webrtc"] = not info["mingwclang"] 82 info["opt"] = ( 83 not info["debug"] and not info["asan"] and not info["tsan"] and not info["ccov"] 84 ) 85 info["os"] = platform_info.os 86 87 # crashreporter is disabled for asan / tsan builds 88 if info["asan"] or info["tsan"]: 89 info["crashreporter"] = False 90 else: 91 info["crashreporter"] = True 92 93 info["appname"] = "fennec" if info["os"] == "android" else "firefox" 94 info["buildapp"] = "browser" 95 96 info["processor"] = platform_info.arch 97 98 # guess toolkit 99 if info["os"] == "android": 100 info["toolkit"] = "android" 101 elif info["os"] == "win": 102 info["toolkit"] = "windows" 103 elif info["os"] == "mac": 104 info["toolkit"] = "cocoa" 105 else: 106 info["toolkit"] = "gtk" 107 info["display"] = platform_info.display or "x11" 108 109 info["os_version"] = platform_info.os_version 110 111 for variant in TEST_VARIANTS: 112 tag = TEST_VARIANTS[variant].get("mozinfo", "") 113 if tag == "": 114 continue 115 116 value = variant in runtime_keys 117 118 if variant == "1proc": 119 value = not value 120 elif "fission" in variant: 121 value = any( 122 "1proc" not in key or "no-fission" not in key for key in runtime_keys 123 ) 124 if "no-fission" not in variant: 125 value = not value 126 elif tag == "xorigin": 127 value = any("xorigin" in key for key in runtime_keys) 128 129 info[tag] = value 130 131 # wpt has canvas and webgpu as tags, lets find those 132 for tag in WPT_SUBSUITES.keys(): 133 if tag in task["test-name"]: 134 info[tag] = True 135 else: 136 info[tag] = False 137 138 # NOTE: as we are using an array here, frozenset() cannot work with a 'list' 139 # this is cast to a string 140 info["tag"] = json.dumps(test_tags) 141 142 info["automation"] = True 143 return info 144 145 146 @memoize 147 def get_runtimes(platform, suite_name): 148 if not suite_name or not platform: 149 raise TypeError("suite_name and platform cannot be empty.") 150 151 base = os.path.join(GECKO, "testing", "runtimes", "manifest-runtimes-{}.json") 152 for key in ("android", "windows"): 153 if key in platform: 154 path = base.format(key) 155 break 156 else: 157 path = base.format("unix") 158 159 if not os.path.exists(path): 160 raise OSError(f"manifest runtime file at {path} not found.") 161 162 with open(path) as fh: 163 return json.load(fh).get(suite_name, {}) 164 165 166 def chunk_manifests(suite, platform, chunks, manifests): 167 """Run the chunking algorithm. 168 169 Args: 170 platform (str): Platform used to find runtime info. 171 chunks (int): Number of chunks to split manifests into. 172 manifests(list): Manifests to chunk. 173 174 Returns: 175 A list of length `chunks` where each item contains a list of manifests 176 that run in that chunk. 177 """ 178 if "web-platform-tests" not in suite: 179 ini_manifests = {x.replace(".toml", ".ini"): x for x in manifests} 180 runtimes = { 181 k: v for k, v in get_runtimes(platform, suite).items() if k in ini_manifests 182 } 183 184 cbr = chunk_by_runtime(None, chunks, runtimes) 185 return [ 186 [ini_manifests.get(m, m) for m in c] 187 for _, c in cbr.get_chunked_manifests(manifests) 188 ] 189 190 # Keep track of test paths for each chunk, and the runtime information. 191 # Spread out the test manifests evenly across all chunks. 192 sorted_manifests = sorted(manifests) 193 chunked_manifests = [sorted_manifests[c::chunks] for c in range(chunks)] 194 return chunked_manifests 195 196 197 class BaseManifestLoader(metaclass=ABCMeta): 198 def __init__(self, params): 199 self.params = params 200 201 @abstractmethod 202 def get_manifests(self, flavor, subsuite, mozinfo): 203 """Compute which manifests should run for the given flavor, subsuite and mozinfo. 204 205 This function returns skipped manifests separately so that more balanced 206 chunks can be achieved by only considering "active" manifests in the 207 chunking algorithm. 208 209 Args: 210 flavor (str): The suite to run. Values are defined by the 'build_flavor' key 211 in `moztest.resolve.TEST_SUITES`. 212 subsuite (str): The subsuite to run or 'undefined' to denote no subsuite. 213 mozinfo (frozenset): Set of data in the form of (<key>, <value>) used 214 for filtering. 215 216 Returns: 217 A tuple of two manifest lists. The first is the set of active manifests (will 218 run at least one test. The second is a list of skipped manifests (all tests are 219 skipped). 220 """ 221 222 223 class DefaultLoader(BaseManifestLoader): 224 """Load manifests using metadata from the TestResolver.""" 225 226 @memoize 227 def get_tests(self, suite): 228 suite_definition = TEST_SUITES[suite] 229 return list( 230 resolver.resolve_tests( 231 flavor=suite_definition["build_flavor"], 232 subsuite=suite_definition.get("kwargs", {}).get( 233 "subsuite", "undefined" 234 ), 235 ) 236 ) 237 238 @memoize 239 def get_manifests(self, suite, frozen_mozinfo): 240 mozinfo = dict(frozen_mozinfo) 241 242 tests = self.get_tests(suite) 243 244 mozinfo_tags = json.loads(mozinfo.get("tag", "[]")) 245 246 if "web-platform-tests" in suite: 247 manifests = set() 248 249 subsuite = next((x for x in WPT_SUBSUITES.keys() if mozinfo.get(x)), None) 250 251 if subsuite: 252 subsuite_paths = WPT_SUBSUITES[subsuite] 253 for t in tests: 254 if mozinfo_tags and not any( 255 x in t.get("tags", []) for x in mozinfo_tags 256 ): 257 continue 258 259 manifest = t["manifest"] 260 if any(x in manifest for x in subsuite_paths): 261 manifests.add(manifest) 262 else: 263 all_subsuite_paths = [ 264 path for paths in WPT_SUBSUITES.values() for path in paths 265 ] 266 for t in tests: 267 if mozinfo_tags and not any( 268 x in t.get("tags", []) for x in mozinfo_tags 269 ): 270 continue 271 272 manifest = t["manifest"] 273 if not any(path in manifest for path in all_subsuite_paths): 274 manifests.add(manifest) 275 276 return { 277 "active": list(manifests), 278 "skipped": [], 279 "other_dirs": {}, 280 } 281 282 filters = [] 283 SUITES_WITHOUT_TAG = { 284 "crashtest", 285 "crashtest-qr", 286 "jsreftest", 287 "reftest", 288 "reftest-qr", 289 } 290 291 # Exclude suites that don't support --tag to prevent manifests from 292 # being optimized out, which would result in no jobs being triggered. 293 # No need to check suites like gtest, as all suites in compiled.yml 294 # have test-manifest-loader set to null, meaning this function is never 295 # called. 296 # Note there's a similar list in desktop_unittest.py in 297 # DesktopUnittest's _query_abs_base_cmd method. The lists should be 298 # kept in sync. 299 assert suite not in ["gtest", "cppunittest", "jittest"] 300 301 if suite not in SUITES_WITHOUT_TAG and mozinfo_tags: 302 filters.extend([tags([x]) for x in mozinfo_tags]) 303 304 m = TestManifest() 305 m.tests = tests 306 active_tests = m.active_tests( 307 disabled=False, exists=False, filters=filters, **mozinfo 308 ) 309 310 active_manifests = {chunk_by_runtime.get_manifest(t) for t in active_tests} 311 312 skipped_manifests = {chunk_by_runtime.get_manifest(t) for t in tests} 313 skipped_manifests.difference_update(active_manifests) 314 return { 315 "active": list(active_manifests), 316 "skipped": list(skipped_manifests), 317 "other_dirs": {}, 318 } 319 320 321 class BugbugLoader(DefaultLoader): 322 """Load manifests using metadata from the TestResolver, and then 323 filter them based on a query to bugbug.""" 324 325 CONFIDENCE_THRESHOLD = CT_LOW 326 327 def __init__(self, *args, **kwargs): 328 super().__init__(*args, **kwargs) 329 self.timedout = False 330 331 @memoize 332 def get_manifests(self, suite, mozinfo): 333 manifests = super().get_manifests(suite, mozinfo) 334 335 # Don't prune any manifests if we're on a backstop push or there was a timeout. 336 if self.params["backstop"] or self.timedout: 337 return manifests 338 339 try: 340 data = push_schedules(self.params["project"], self.params["head_rev"]) 341 except (BugbugTimeoutException, RetryError): 342 traceback.print_exc() 343 logger.warning("Timed out waiting for bugbug, loading all test manifests.") 344 self.timedout = True 345 return self.get_manifests(suite, mozinfo) 346 347 bugbug_manifests = { 348 m 349 for m, c in data.get("groups", {}).items() 350 if c >= self.CONFIDENCE_THRESHOLD 351 } 352 353 manifests["active"] = list(set(manifests["active"]) & bugbug_manifests) 354 manifests["skipped"] = list(set(manifests["skipped"]) & bugbug_manifests) 355 return manifests 356 357 358 manifest_loaders = { 359 "bugbug": BugbugLoader, 360 "default": DefaultLoader, 361 } 362 363 _loader_cache = {} 364 365 366 def get_manifest_loader(name, params): 367 # Ensure we never create more than one instance of the same loader type for 368 # performance reasons. 369 if name in _loader_cache: 370 return _loader_cache[name] 371 372 loader = manifest_loaders[name](dict(params)) 373 _loader_cache[name] = loader 374 return loader