strategies.py (3783B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import datetime 7 import logging 8 9 import mozpack.path as mozpath 10 from mozbuild.base import MozbuildObject 11 from mozbuild.util import memoize 12 from taskgraph.optimize.base import OptimizationStrategy, register_strategy 13 from taskgraph.optimize.strategies import IndexSearch 14 from taskgraph.util.parameterization import resolve_timestamps 15 16 from gecko_taskgraph.optimize.mozlint import SkipUnlessMozlint 17 18 logger = logging.getLogger(__name__) 19 20 21 @register_strategy("skip-unless-schedules") 22 class SkipUnlessSchedules(OptimizationStrategy): 23 @memoize 24 def scheduled_by_push(self, files_changed): 25 mbo = MozbuildObject.from_environment() 26 # the decision task has a sparse checkout, so, mozbuild_reader will use 27 # a MercurialRevisionFinder with revision '.', which should be the same 28 # as `revision`; in other circumstances, it will use a default reader 29 rdr = mbo.mozbuild_reader(config_mode="empty") 30 31 components = set() 32 for p, m in rdr.files_info(files_changed).items(): 33 components |= set(m["SCHEDULES"].components) 34 35 return components 36 37 def should_remove_task(self, task, params, conditions): 38 if params.get("pushlog_id") == -1: 39 return False 40 41 scheduled = self.scheduled_by_push(frozenset(params["files_changed"])) 42 conditions = set(conditions) 43 # if *any* of the condition components are scheduled, do not optimize 44 if conditions & scheduled: 45 return False 46 47 return True 48 49 50 @register_strategy("skip-unless-has-relevant-tests") 51 class SkipUnlessHasRelevantTests(OptimizationStrategy): 52 """Optimizes tasks that don't run any tests that were 53 in child directories of a modified file. 54 """ 55 56 @memoize 57 def get_changed_dirs(self, files_changed): 58 changed = map(mozpath.dirname, files_changed) 59 # Filter out empty directories (from files modified in the root). 60 # Otherwise all tasks would be scheduled. 61 return {d for d in changed if d} 62 63 def should_remove_task(self, task, params, _): 64 if not task.attributes.get("test_manifests"): 65 return True 66 67 for d in self.get_changed_dirs(frozenset(params["files_changed"])): 68 for t in task.attributes["test_manifests"]: 69 if t.startswith(d): 70 logger.debug( 71 f"{task.label} runs a test path ({t}) contained by a modified file ({d})" 72 ) 73 return False 74 return True 75 76 77 register_strategy("skip-unless-mozlint", args=("tools/lint",))(SkipUnlessMozlint) 78 79 80 @register_strategy("skip-unless-missing") 81 class SkipUnlessMissing(OptimizationStrategy): 82 """Skips a task unless it is missing from a specified index. 83 84 This simply defers to Taskgraph's `index-search` optimization. The reason 85 we need this shim is because replacement and removal optimizations can't be 86 joined together in a composite strategy as removal and replacement happen 87 at different times. 88 """ 89 90 index_search = IndexSearch() 91 92 def _convert_datetime_str(self, dt): 93 if dt.endswith("Z"): 94 dt = dt[:-1] 95 96 return datetime.datetime.fromisoformat(dt).strftime(self.index_search.fmt) 97 98 def should_remove_task(self, task, params, index): 99 now = datetime.datetime.now(datetime.timezone.utc) 100 deadline = self._convert_datetime_str( 101 resolve_timestamps(now, task.task["deadline"]) 102 ) 103 return bool( 104 self.index_search.should_replace_task(task, params, deadline, [index]) 105 )