writeruntimes (8026B)
1 #!/bin/sh 2 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- 3 # vim: set filetype=python: 4 5 # This Source Code Form is subject to the terms of the Mozilla Public 6 # License, v. 2.0. If a copy of the MPL was not distributed with this 7 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 9 # The beginning of this script is both valid shell and valid python, 10 # such that the script starts with the shell and is reexecuted python 11 ''':' 12 which mach > /dev/null 2>&1 && exec mach python "$0" "$@" || 13 echo "mach not found, either add it to your \$PATH or run this script via ./mach python testing/runtimes/writeruntimes"; exit # noqa 14 ''' 15 16 import datetime 17 import json 18 import os 19 import sys 20 import time 21 from argparse import ArgumentParser 22 from collections import defaultdict 23 24 import requests 25 26 from moztest.resolve import ( 27 TestManifestLoader, 28 TestResolver, 29 TEST_SUITES, 30 ) 31 32 here = os.path.abspath(os.path.dirname(__file__)) 33 ACTIVE_DATA_URL = "https://activedata.allizom.org/query" 34 EXCEED_LIMIT = [ 35 # Suites that exceed 10,000 ActiveData result limit will be defined here. 36 'web-platform-tests', 37 'web-platform-tests-reftest', 38 ] 39 MAX_RETRIES = 10 40 RETRY_INTERVAL = 10 41 42 43 def construct_query(suite, platform): 44 if platform in ('windows', 'android'): 45 platform_clause = '{"find":{"run.machine.platform": "%s"}}' % platform 46 else: 47 # Bundle macosx and linux results together - they are not too different. 48 platform_clause = ''' 49 { 50 "not": { 51 "or": [ 52 {"find":{"run.machine.platform": "windows"}}, 53 {"find":{"run.machine.platform": "android"}} 54 ] 55 } 56 } 57 ''' 58 59 # Only use this if the suite being queried exceeeds 10,000 results. 60 output_clause = '"destination": "url",\n"format": "list",' if suite in EXCEED_LIMIT else '' 61 62 query = """ 63 { 64 "from":"unittest", 65 "limit":200000, 66 "groupby":["result.test"], 67 "select":{"value":"result.duration","aggregate":"median"}, 68 %s 69 "where":{"and":[ 70 {"eq":{"repo.branch.name": "mozilla-central"}}, 71 {"in":{"result.status": ["OK", "PASS", "FAIL"]}}, 72 {"gt":{"run.timestamp": {"date": "today-week"}}}, 73 {"eq":{"run.suite.fullname":"%s"}}, 74 %s 75 ]} 76 } 77 """ % (output_clause, suite, platform_clause) 78 79 return query 80 81 82 def query_activedata(suite, platform): 83 query = construct_query(suite, platform) 84 print("Querying ActiveData for '{}' tests on '{}' platforms.. " 85 .format(suite, platform), end='') 86 sys.stdout.flush() 87 response = requests.post(ACTIVE_DATA_URL, 88 data=query, 89 stream=True) 90 response.raise_for_status() 91 92 # Presence of destination clause in the query requires additional processing 93 # to produce the dataset that can be used. 94 if suite in EXCEED_LIMIT: 95 # The output_url is where result of the query will be stored. 96 output_url = response.json()["url"] 97 98 tried = 0 99 while tried < MAX_RETRIES: 100 # Use the requests.Session object to manage requests, since the output_url 101 # can often return 403 Forbidden. 102 session = requests.Session() 103 response = session.get(output_url) 104 if response.status_code == 200: 105 break 106 # A non-200 status code means we should retry after some wait. 107 time.sleep(RETRY_INTERVAL) 108 tried += 1 109 110 # Data returned from destination is in format of: 111 # {data: [result: {test: test_name, duration: duration}]} 112 # Normalize it to the format expected by compute_manifest_runtimes. 113 raw_data = response.json()["data"] 114 data = dict([[item['result']['test'], item['result']['duration']] for item in raw_data]) 115 else: 116 data = dict(response.json()["data"]) 117 118 print("{} found".format(len(data))) 119 return data 120 121 122 def write_runtimes(manifest_runtimes, platform, suite, outdir=here): 123 if not os.path.exists(outdir): 124 os.makedirs(outdir) 125 126 outfilename = os.path.join(outdir, "manifest-runtimes-{}.json".format(platform)) 127 # If file is not present, initialize a file with empty JSON object. 128 if not os.path.exists(outfilename): 129 with open(outfilename, 'w+') as f: 130 json.dump({}, f) 131 132 # Load the entire file. 133 with open(outfilename, 'r') as f: 134 data = json.load(f) 135 136 # Update the specific suite with the new runtime information and write to file. 137 data[suite] = manifest_runtimes 138 with open(outfilename, 'w') as f: 139 json.dump(data, f, indent=2, sort_keys=True) 140 141 142 def compute_manifest_runtimes(suite, platform): 143 resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) 144 145 crashtest_prefixes = { 146 'http': '/tests/', 147 'chrome': '/reftest/content/', 148 'file': '/reftest/tests/', 149 } 150 manifest_runtimes = defaultdict(float) 151 data = query_activedata(suite, platform) 152 153 if "web-platform-tests" in suite: 154 wpt_groups = {t["name"]: t["manifest"] 155 for t in resolver.resolve_tests(flavor="web-platform-tests")} 156 157 for path, duration in data.items(): 158 # Returned data did not contain a test path, so go to next result. 159 if not path: 160 continue 161 162 if suite in ('reftest', 'crashtest') and ' ' in path: 163 path = path.split()[0] 164 165 if suite == 'crashtest' and '://' in path: 166 # Crashtest paths are URLs with various schemes and prefixes. 167 # Normalize it to become relative to mozilla-central. 168 scheme = path[:path.index('://')] 169 if ':' in scheme: 170 scheme = scheme.split(':')[-1] 171 prefix = crashtest_prefixes[scheme] 172 path = path.split(prefix, 1)[-1] 173 elif suite == 'xpcshell' and ':' in path: 174 path = path.split(':', 1)[-1] 175 176 if "web-platform-tests" in suite: 177 if path in wpt_groups: 178 manifest_runtimes[wpt_groups[path]] += duration 179 continue 180 181 if path not in resolver.tests_by_path: 182 continue 183 184 for test in resolver.tests_by_path[path]: 185 manifest = test.get('ancestor_manifest') or test['manifest_relpath'] 186 manifest_runtimes[manifest] += duration 187 188 manifest_runtimes = {k: round(v, 2) for k, v in manifest_runtimes.items()} 189 return manifest_runtimes 190 191 192 def cli(args=sys.argv[1:]): 193 default_suites = [suite for suite, obj in TEST_SUITES.items() if 'build_flavor' in obj] 194 default_platforms = ['android', 'windows', 'unix'] 195 196 parser = ArgumentParser() 197 parser.add_argument('-o', '--output-directory', dest='outdir', default=here, 198 help="Directory to save runtime data.") 199 parser.add_argument('-s', '--suite', dest='suites', action='append', 200 default=None, choices=default_suites, 201 help="Suite(s) to include in the data set (default: all)") 202 parser.add_argument('-p', '--platform', dest='platforms', action='append', 203 default=None, choices=default_platforms, 204 help="Platform(s) to gather runtime information on " 205 "(default: all).") 206 args = parser.parse_args(args) 207 208 # If a suite was specified, use that. Otherwise, use the full default set. 209 suites = args.suites or default_suites 210 # Same as above, but for the platform clause. 211 platforms = args.platforms or default_platforms 212 213 for platform in platforms: 214 for suite in suites: 215 runtimes = compute_manifest_runtimes(suite, platform) 216 if not runtimes: 217 print("Not writing runtime data for '{}' for '{}' as no data was found".format(suite, platform)) 218 continue 219 220 write_runtimes(runtimes, platform, suite, outdir=args.outdir) 221 222 223 if __name__ == "__main__": 224 sys.exit(cli())