gecko_profile.py (8261B)
1 # This Source Code Form is subject to the terms of the Mozilla Public 2 # License, v. 2.0. If a copy of the MPL was not distributed with this 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5 6 import logging 7 8 import requests 9 from taskcluster.exceptions import TaskclusterRestFailure 10 from taskgraph.util.taskcluster import get_artifact_from_index, get_task_definition 11 12 from .registry import register_callback_action 13 from .util import combine_task_graph_files, create_tasks, fetch_graph_and_labels 14 15 PUSHLOG_TMPL = "{}/json-pushes?version=2&startID={}&endID={}" 16 INDEX_TMPL = "gecko.v2.{}.pushlog-id.{}.decision" 17 18 logger = logging.getLogger(__name__) 19 20 21 @register_callback_action( 22 title="GeckoProfile", 23 name="geckoprofile", 24 symbol="Gp", 25 description=( 26 "Take the label of the current task, " 27 "and trigger the task with that label " 28 "on previous pushes in the same project " 29 "while adding the --gecko-profile cmd arg. " 30 "Plus optional overrides for threads, " 31 "features, and sampling interval." 32 ), 33 order=200, 34 context=[ 35 {"test-type": "talos"}, 36 {"test-type": "raptor"}, 37 {"test-type": "mozperftest"}, 38 ], 39 schema={ 40 "type": "object", 41 "properties": { 42 "depth": { 43 "type": "integer", 44 "default": 1, 45 "minimum": 1, 46 "maximum": 10, 47 "title": "Depth", 48 "description": "How many pushes to backfill the profiling task on.", 49 }, 50 "gecko_profile_interval": { 51 "type": "integer", 52 "default": None, 53 "title": "Sampling interval (ms)", 54 "description": "How often to sample the profiler (in ms).", 55 }, 56 "gecko_profile_features": { 57 "type": "string", 58 "default": "", 59 "title": "Features", 60 "description": "Comma-separated Gecko profiler features. " 61 "Example: js,stackwalk,cpu,screenshots,memory", 62 }, 63 "gecko_profile_threads": { 64 "type": "string", 65 "default": "", 66 "title": "Threads", 67 "description": "Comma-separated thread names to profile. " 68 "Example: GeckoMain,Compositor,Renderer", 69 }, 70 }, 71 }, 72 available=lambda parameters: True, 73 ) 74 def geckoprofile_action(parameters, graph_config, input, task_group_id, task_id): 75 task = get_task_definition(task_id) 76 label = task["metadata"]["name"] 77 pushes = [] 78 depth = input.get("depth", 1) 79 end_id = int(parameters["pushlog_id"]) 80 81 while True: 82 start_id = max(end_id - depth, 0) 83 pushlog_url = PUSHLOG_TMPL.format( 84 parameters["head_repository"], start_id, end_id 85 ) 86 r = requests.get(pushlog_url) 87 r.raise_for_status() 88 pushes = pushes + list(r.json()["pushes"].keys()) 89 if len(pushes) >= depth: 90 break 91 92 end_id = start_id - 1 93 start_id -= depth 94 if start_id < 0: 95 break 96 97 pushes = sorted(pushes)[-depth:] 98 backfill_pushes = [] 99 100 for push in pushes: 101 try: 102 push_params = get_artifact_from_index( 103 INDEX_TMPL.format(parameters["project"], push), "public/parameters.yml" 104 ) 105 push_decision_task_id, full_task_graph, label_to_taskid, _ = ( 106 fetch_graph_and_labels(push_params, graph_config) 107 ) 108 except TaskclusterRestFailure as e: 109 logger.info(f"Skipping {push} due to missing index artifacts! Error: {e}") 110 continue 111 112 if label in full_task_graph.tasks.keys(): 113 114 def modifier(task): 115 if task.label != label: 116 return task 117 118 interval = input.get("gecko_profile_interval") 119 features = input.get("gecko_profile_features") 120 threads = input.get("gecko_profile_threads") 121 122 task_kind = task.kind 123 env = task.task["payload"]["env"] 124 perf_flags = env.get("PERF_FLAGS", "") 125 test_suite = task.attributes.get("unittest_suite") 126 profiling_command_flags = ["--gecko-profile"] 127 128 if task_kind == "perftest": 129 # Add "gecko-profile" to PERF_FLAGS if missing and then add remaining 130 # Gecko Profiler customizations via MOZ_PROFILER_STARTUP_* env overrides. 131 if "gecko-profile" not in perf_flags: 132 env["PERF_FLAGS"] = (perf_flags + " gecko-profile").strip() 133 134 if interval is not None: 135 env["MOZ_PROFILER_STARTUP_INTERVAL"] = str(interval) 136 if features is not None: 137 env["MOZ_PROFILER_STARTUP_FEATURES"] = features 138 if threads is not None: 139 env["MOZ_PROFILER_STARTUP_FILTERS"] = threads 140 141 elif test_suite == "raptor": 142 # Use PERF_FLAGS env to cusomize profiler settings. 143 raptor_flags = [] 144 if interval is not None: 145 raptor_flags.append(f"gecko-profile-interval={interval}") 146 if features is not None: 147 raptor_flags.append(f"gecko-profile-features={features}") 148 if threads is not None: 149 raptor_flags.append(f"gecko-profile-threads={threads}") 150 151 env["PERF_FLAGS"] = ( 152 perf_flags + " " + " ".join(raptor_flags) 153 ).strip() 154 155 elif test_suite == "talos": 156 # Pass everything through the command directly 157 # Bug 1979192 will modify Talos to make use of PERF_FLAGS. 158 if interval is not None: 159 profiling_command_flags.append( 160 f"--gecko-profile-interval={interval}" 161 ) 162 if features is not None: 163 profiling_command_flags.append( 164 f"--gecko-profile-features={features}" 165 ) 166 if threads is not None: 167 profiling_command_flags.append( 168 f"--gecko-profile-threads={threads}" 169 ) 170 171 if "command" in task.task["payload"]: 172 cmd = task.task["payload"]["command"] 173 task.task["payload"]["command"] = add_args_to_perf_command( 174 cmd, profiling_command_flags 175 ) 176 177 task.task["extra"]["treeherder"]["symbol"] += "-p" 178 task.task["extra"]["treeherder"]["groupName"] += " (profiling)" 179 return task 180 181 create_tasks( 182 graph_config, 183 [label], 184 full_task_graph, 185 label_to_taskid, 186 push_params, 187 push_decision_task_id, 188 push, 189 modifier=modifier, 190 ) 191 backfill_pushes.append(push) 192 else: 193 logger.info(f"Could not find {label} on {push}. Skipping.") 194 combine_task_graph_files(backfill_pushes) 195 196 197 def add_args_to_perf_command(payload_commands, extra_args=()): 198 """ 199 Add custom command line args to a given command. 200 args: 201 payload_commands: the raw command as seen by taskcluster 202 extra_args: array of args we want to inject 203 """ 204 perf_command_idx = -1 # currently, it's the last (or only) command 205 perf_command = payload_commands[perf_command_idx] 206 207 command_form = "default" 208 if isinstance(perf_command, str): 209 # windows has a single command, in long string form 210 perf_command = perf_command.split(" ") 211 command_form = "string" 212 # osx & linux have an array of subarrays 213 214 perf_command.extend(extra_args) 215 216 if command_form == "string": 217 # pack it back to list 218 perf_command = " ".join(perf_command) 219 220 payload_commands[perf_command_idx] = perf_command 221 222 return payload_commands