confirm_failure.py (9556B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import logging 7 from functools import partial 8 9 from taskgraph.util import json 10 from taskgraph.util.taskcluster import get_artifact, get_task_definition, list_artifacts 11 12 from ..util.constants import TEST_KINDS 13 from .registry import register_callback_action 14 from .retrigger import retrigger_action 15 from .util import add_args_to_command, create_tasks, fetch_graph_and_labels 16 17 logger = logging.getLogger(__name__) 18 19 20 def get_failures(task_id, task_definition): 21 """Returns a dict containing properties containing a list of 22 directories containing test failures and a separate list of 23 individual test failures from the errorsummary.log artifact for 24 the task. 25 26 Find test path to pass to the task in 27 MOZHARNESS_TEST_PATHS. If no appropriate test path can be 28 determined, nothing is returned. 29 """ 30 31 def fix_wpt_name(test): 32 # TODO: find other cases to handle 33 if ".any." in test: 34 test = "%s.any.js" % test.split(".any.")[0] 35 if ".window.html" in test: 36 test = test.replace(".window.html", ".window.js") 37 38 if test.startswith("/_mozilla"): 39 test = "testing/web-platform/mozilla/tests" + test[len("/_mozilla") :] 40 else: 41 test = "testing/web-platform/tests/" + test.strip("/") 42 # some wpt tests have params, those are not supported 43 test = test.split("?")[0] 44 45 return test 46 47 # collect dirs that don't have a specific manifest 48 dirs = [] 49 tests = [] 50 51 artifacts = list_artifacts(task_id) 52 for artifact in artifacts: 53 if "name" not in artifact or not artifact["name"].endswith("errorsummary.log"): 54 continue 55 56 stream = get_artifact(task_id, artifact["name"]) 57 if not stream: 58 continue 59 60 # We handle the stream as raw bytes because it may contain invalid 61 # UTF-8 characters in portions other than those containing the error 62 # messages we're looking for. 63 for line in stream.read().split(b"\n"): 64 if not line.strip(): 65 continue 66 67 l = json.loads(line) 68 if "group_results" in l.keys() and l["status"] != "OK": 69 dirs.append(l["group_results"].group()) 70 71 elif "test" in l.keys(): 72 if not l["test"]: 73 print("Warning: no testname in errorsummary line: %s" % l) 74 continue 75 76 test_path = l["test"].split(" ")[0] 77 found_path = False 78 79 # tests with url params (wpt), will get confused here 80 if "?" not in test_path: 81 test_path = test_path.split(":")[-1] 82 83 # edge case where a crash on shutdown has a "test" name == group name 84 if ( 85 test_path.endswith(".toml") 86 or test_path.endswith(".ini") 87 or test_path.endswith(".list") 88 ): 89 # TODO: consider running just the manifest 90 continue 91 92 # edge cases with missing test names 93 if ( 94 test_path is None 95 or test_path == "None" 96 or "SimpleTest" in test_path 97 ): 98 continue 99 100 if "signature" in l.keys(): 101 # dealing with a crash 102 found_path = True 103 if "web-platform" in task_definition["extra"]["suite"]: 104 test_path = fix_wpt_name(test_path) 105 else: 106 if "status" not in l and "expected" not in l: 107 continue 108 109 if l["status"] != l["expected"]: 110 if l["status"] not in l.get("known_intermittent", []): 111 found_path = True 112 if "web-platform" in task_definition["extra"]["suite"]: 113 test_path = fix_wpt_name(test_path) 114 115 if found_path and test_path: 116 fpath = test_path.replace("\\", "/") 117 tval = {"path": fpath, "group": l["group"]} 118 # only store one failure per test 119 if not [t for t in tests if t["path"] == fpath]: 120 tests.append(tval) 121 122 # only run the failing test not both test + dir 123 if l["group"] in dirs: 124 dirs.remove(l["group"]) 125 126 # TODO: 10 is too much; how to get only NEW failures? 127 if len(tests) > 10: 128 break 129 130 dirs = [{"path": "", "group": d} for d in list(set(dirs))] 131 return {"dirs": dirs, "tests": tests} 132 133 134 def get_repeat_args(task_definition, failure_group): 135 task_name = task_definition["metadata"]["name"] 136 repeatable_task = False 137 if ( 138 "crashtest" in task_name 139 or "mochitest" in task_name 140 or "reftest" in task_name 141 or "xpcshell" in task_name 142 or "web-platform" in task_name 143 and "jsreftest" not in task_name 144 ): 145 repeatable_task = True 146 147 repeat_args = "" 148 if not repeatable_task: 149 return repeat_args 150 151 if failure_group == "dirs": 152 # execute 3 total loops 153 repeat_args = ["--repeat=2"] if repeatable_task else [] 154 elif failure_group == "tests": 155 # execute 5 total loops 156 repeat_args = ["--repeat=4"] if repeatable_task else [] 157 158 return repeat_args 159 160 161 def confirm_modifier(task, input): 162 if task.label != input["label"]: 163 return task 164 165 logger.debug(f"Modifying paths for {task.label}") 166 167 # If the original task has defined test paths 168 suite = input.get("suite") 169 test_path = input.get("test_path") 170 test_group = input.get("test_group") 171 if test_path or test_group: 172 repeat_args = input.get("repeat_args") 173 174 if repeat_args: 175 task.task["payload"]["command"] = add_args_to_command( 176 task.task["payload"]["command"], extra_args=repeat_args 177 ) 178 179 # TODO: do we need this attribute? 180 task.attributes["test_path"] = test_path 181 182 task.task["payload"]["env"]["MOZHARNESS_TEST_PATHS"] = json.dumps( 183 {suite: [test_group]}, sort_keys=True 184 ) 185 task.task["payload"]["env"]["MOZHARNESS_CONFIRM_PATHS"] = json.dumps( 186 {suite: [test_path]}, sort_keys=True 187 ) 188 task.task["payload"]["env"]["MOZLOG_DUMP_ALL_TESTS"] = "1" 189 190 task.task["metadata"]["name"] = task.label 191 task.task["tags"]["action"] = "confirm-failure" 192 return task 193 194 195 @register_callback_action( 196 name="confirm-failures", 197 title="Confirm failures in job", 198 symbol="cf", 199 description="Re-run Tests for original manifest, directories or tests for failing tests.", 200 order=150, 201 context=[{"kind": kind} for kind in TEST_KINDS], 202 schema={ 203 "type": "object", 204 "properties": { 205 "label": {"type": "string", "description": "A task label"}, 206 "suite": {"type": "string", "description": "Test suite"}, 207 "test_path": {"type": "string", "description": "A full path to test"}, 208 "test_group": { 209 "type": "string", 210 "description": "A full path to group name", 211 }, 212 "repeat_args": { 213 "type": "string", 214 "description": "args to pass to test harness", 215 }, 216 }, 217 "additionalProperties": False, 218 }, 219 ) 220 def confirm_failures(parameters, graph_config, input, task_group_id, task_id): 221 task_definition = get_task_definition(task_id) 222 decision_task_id, full_task_graph, label_to_taskid, _ = fetch_graph_and_labels( 223 parameters, graph_config 224 ) 225 226 # create -cf label; ideally make this a common function 227 task_definition["metadata"]["name"].split("-") 228 cfname = "%s-cf" % task_definition["metadata"]["name"] 229 230 if cfname not in full_task_graph.tasks: 231 raise Exception(f"{cfname} was not found in the task-graph") 232 233 to_run = [cfname] 234 235 suite = task_definition["extra"]["suite"] 236 if "-coverage" in suite: 237 suite = suite[: suite.index("-coverage")] 238 if "-qr" in suite: 239 suite = suite[: suite.index("-qr")] 240 failures = get_failures(task_id, task_definition) 241 242 if failures["dirs"] == [] and failures["tests"] == []: 243 logger.info("need to retrigger task as no specific test failures found") 244 retrigger_action(parameters, graph_config, input, decision_task_id, task_id) 245 return 246 247 # for each unique failure, create a new confirm failure job 248 for failure_group in failures: 249 for failure_path in failures[failure_group]: 250 repeat_args = get_repeat_args(task_definition, failure_group) 251 252 input = { 253 "label": cfname, 254 "suite": suite, 255 "test_path": failure_path["path"], 256 "test_group": failure_path["group"], 257 "repeat_args": repeat_args, 258 } 259 260 logger.info("confirm_failures: %s" % failures) 261 create_tasks( 262 graph_config, 263 to_run, 264 full_task_graph, 265 label_to_taskid, 266 parameters, 267 decision_task_id, 268 modifier=partial(confirm_modifier, input=input), 269 )