perfile.py (3369B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import itertools 7 import logging 8 import math 9 10 import taskgraph 11 from mozbuild.util import memoize 12 from mozpack.path import match as mozpackmatch 13 from taskgraph.util import json 14 15 logger = logging.getLogger(__name__) 16 17 18 @memoize 19 def perfile_number_of_chunks(is_try, try_task_config, files_changed, type): 20 changed_files = set(files_changed) 21 if taskgraph.fast and not is_try: 22 # When iterating on taskgraph changes, the exact number of chunks that 23 # test-verify runs usually isn't important, so skip it when going fast. 24 return 3 25 tests_per_chunk = 10.0 26 if type.startswith("test-coverage"): 27 tests_per_chunk = 30.0 28 29 if type.startswith("test-verify-wpt") or type.startswith("test-coverage-wpt"): 30 file_patterns = [ 31 "testing/web-platform/tests/**", 32 "testing/web-platform/mozilla/tests/**", 33 ] 34 elif type.startswith("test-verify-gpu") or type.startswith("test-coverage-gpu"): 35 file_patterns = [ 36 "**/*webgl*/**/test_*", 37 "**/dom/canvas/**/test_*", 38 "**/gfx/tests/**/test_*", 39 "**/devtools/canvasdebugger/**/browser_*", 40 "**/reftest*/**", 41 ] 42 elif type.startswith("test-verify") or type.startswith("test-coverage"): 43 file_patterns = [ 44 "**/test_*", 45 "**/browser_*", 46 "**/crashtest*/**", 47 "js/src/tests/test/**", 48 "js/src/tests/non262/**", 49 "js/src/tests/test262/**", 50 ] 51 else: 52 # Returning 0 means no tests to run, this captures non test-verify tasks 53 return 1 54 55 if try_task_config: 56 suite_to_paths = json.loads(try_task_config) 57 specified_files = itertools.chain.from_iterable(suite_to_paths.values()) 58 changed_files.update(specified_files) 59 60 test_count = 0 61 for pattern in file_patterns: 62 for path in changed_files: 63 # TODO: consider running tests if a manifest changes 64 if path.endswith(".list") or path.endswith(".ini"): 65 continue 66 if path.endswith("^headers^"): 67 continue 68 69 if mozpackmatch(path, pattern): 70 gpu = False 71 if type in {"test-verify-e10s", "test-coverage-e10s"}: 72 # file_patterns for test-verify will pick up some gpu tests, lets ignore 73 # in the case of reftest, we will not have any in the regular case 74 gpu_dirs = [ 75 "dom/canvas", 76 "gfx/tests", 77 "devtools/canvasdebugger", 78 "webgl", 79 ] 80 for gdir in gpu_dirs: 81 if len(path.split(gdir)) > 1: 82 gpu = True 83 84 if not gpu: 85 test_count += 1 86 87 chunks = test_count / tests_per_chunk 88 chunks = int(math.ceil(chunks)) 89 90 # Never return 0 chunks on try, so that per-file tests can be pushed to try with 91 # an explicit path, and also so "empty" runs can be checked on try. 92 if is_try and chunks == 0: 93 chunks = 1 94 95 return chunks