parsemark.py (7356B)
1 #!/usr/bin/env python 2 3 """%prog [options] shellpath dirpath 4 5 Pulls performance data on parsing via the js shell. 6 Displays the average number of milliseconds it took to parse each file. 7 8 For comparison, something apparently approximating a t-test is performed: 9 "Faster" means that: 10 11 t_baseline_goodrun = (t_baseline_avg - t_baseline_stddev) 12 t_current_badrun = (t_current_avg + t_current_stddev) 13 t_current_badrun < t_baseline_goodrun 14 15 Effectively, a bad run from the current data is better than a good run from the 16 baseline data, we're probably faster. A similar computation is used for 17 determining the "slower" designation. 18 19 Arguments: 20 shellpath executable JavaScript shell 21 dirpath directory filled with parsilicious js files 22 """ 23 24 import json 25 import math 26 import optparse 27 import os 28 import subprocess as subp 29 import sys 30 from string import Template 31 32 try: 33 import compare_bench 34 except ImportError: 35 compare_bench = None 36 37 38 _DIR = os.path.dirname(__file__) 39 JS_CODE_TEMPLATE = Template( 40 """ 41 if (typeof snarf !== 'undefined') read = snarf 42 var contents = read("$filepath"); 43 $prepare 44 for (var i = 0; i < $warmup_run_count; i++) 45 $func(contents, $options); 46 var results = []; 47 for (var i = 0; i < $real_run_count; i++) { 48 var start = elapsed() / 1000; 49 $func(contents, $options); 50 var end = elapsed() / 1000; 51 results.push(end - start); 52 } 53 print(results); 54 """ 55 ) 56 57 58 def gen_filepaths(dirpath, target_ext=".js"): 59 for filename in os.listdir(dirpath): 60 if filename.endswith(target_ext): 61 yield os.path.join(dirpath, filename) 62 63 64 def avg(seq): 65 return sum(seq) / len(seq) 66 67 68 def stddev(seq, mean): 69 diffs = ((float(item) - mean) ** 2 for item in seq) 70 return math.sqrt(sum(diffs) / len(seq)) 71 72 73 def bench( 74 shellpath, filepath, warmup_runs, counted_runs, prepare, func, options, stfu=False 75 ): 76 """Return a list of milliseconds for the counted runs.""" 77 assert '"' not in filepath 78 code = JS_CODE_TEMPLATE.substitute( 79 filepath=filepath, 80 warmup_run_count=warmup_runs, 81 real_run_count=counted_runs, 82 prepare=prepare, 83 func=func, 84 options=options, 85 ) 86 proc = subp.Popen([shellpath, "-e", code], stdout=subp.PIPE) 87 stdout, _ = proc.communicate() 88 milliseconds = [float(val) for val in stdout.decode().split(",")] 89 mean = avg(milliseconds) 90 sigma = stddev(milliseconds, mean) 91 if not stfu: 92 print("Runs:", [int(ms) for ms in milliseconds]) 93 print("Mean:", mean) 94 print(f"Stddev: {sigma:.2f} ({sigma / mean * 100:.2f}% of mean)") 95 return mean, sigma 96 97 98 def parsemark(filepaths, fbench, stfu=False): 99 """:param fbench: fbench(filename) -> float""" 100 bench_map = {} # {filename: (avg, stddev)} 101 for filepath in filepaths: 102 filename = os.path.split(filepath)[-1] 103 if not stfu: 104 print(f"Parsemarking {filename}...") 105 bench_map[filename] = fbench(filepath) 106 print("{") 107 for i, (filename, (avg, stddev)) in enumerate(iter(bench_map.items())): 108 assert '"' not in filename 109 fmt = ' {:30s}: {{"average_ms": {:6.2f}, "stddev_ms": {:6.2f}}}' 110 if i != len(bench_map) - 1: 111 fmt += "," 112 filename_str = f'"{filename}"' 113 print(fmt.format(filename_str, avg, stddev)) 114 print("}") 115 return dict( 116 (filename, dict(average_ms=avg, stddev_ms=stddev)) 117 for filename, (avg, stddev) in iter(bench_map.items()) 118 ) 119 120 121 def main(): 122 parser = optparse.OptionParser(usage=__doc__.strip()) 123 parser.add_option( 124 "-w", 125 "--warmup-runs", 126 metavar="COUNT", 127 type=int, 128 default=5, 129 help="used to minimize test instability [%default]", 130 ) 131 parser.add_option( 132 "-c", 133 "--counted-runs", 134 metavar="COUNT", 135 type=int, 136 default=50, 137 help="timed data runs that count towards the average [%default]", 138 ) 139 parser.add_option( 140 "-s", 141 "--shell", 142 metavar="PATH", 143 help="explicit shell location; when omitted, will look in likely places", 144 ) 145 parser.add_option( 146 "-b", 147 "--baseline", 148 metavar="JSON_PATH", 149 dest="baseline_path", 150 help="json file with baseline values to compare against", 151 ) 152 parser.add_option( 153 "--mode", 154 dest="mode", 155 type="choice", 156 choices=("parse", "dumpStencil", "compile", "decode"), 157 default="parse", 158 help="The target of the benchmark (parse/dumpStencil/compile/decode), defaults to parse", 159 ) 160 parser.add_option( 161 "--lazy", 162 dest="lazy", 163 action="store_true", 164 default=False, 165 help="Use lazy parsing when compiling", 166 ) 167 parser.add_option( 168 "-q", 169 "--quiet", 170 dest="stfu", 171 action="store_true", 172 default=False, 173 help="only print JSON to stdout [%default]", 174 ) 175 options, args = parser.parse_args() 176 try: 177 shellpath = args.pop(0) 178 except IndexError: 179 parser.print_help() 180 print() 181 print("error: shellpath required", file=sys.stderr) 182 return -1 183 try: 184 dirpath = args.pop(0) 185 except IndexError: 186 parser.print_help() 187 print() 188 print("error: dirpath required", file=sys.stderr) 189 return -1 190 if not shellpath or not os.path.exists(shellpath): 191 print("error: could not find shell:", shellpath, file=sys.stderr) 192 return -1 193 if options.baseline_path: 194 if not os.path.isfile(options.baseline_path): 195 print("error: baseline file does not exist", file=sys.stderr) 196 return -1 197 if not compare_bench: 198 print( 199 "error: JSON support is missing, cannot compare benchmarks", 200 file=sys.stderr, 201 ) 202 return -1 203 204 if options.lazy and options.mode == "parse": 205 print( 206 "error: parse mode doesn't support lazy", 207 file=sys.stderr, 208 ) 209 return -1 210 211 funcOpt = {} 212 if options.mode == "decode": 213 encodeOpt = {} 214 encodeOpt["execute"] = False 215 encodeOpt["saveBytecodeWithDelazifications"] = True 216 if not options.lazy: 217 encodeOpt["forceFullParse"] = True 218 219 # In order to test the decoding, we first have to encode the content. 220 prepare = Template( 221 """ 222 contents = cacheEntry(contents); 223 evaluate(contents, $options); 224 """ 225 ).substitute(options=json.dumps(encodeOpt)) 226 227 func = "evaluate" 228 funcOpt["execute"] = False 229 funcOpt["loadBytecode"] = True 230 if not options.lazy: 231 funcOpt["forceFullParse"] = True 232 else: 233 prepare = "" 234 func = options.mode 235 if not options.lazy: 236 funcOpt["forceFullParse"] = True 237 238 def benchfile(filepath): 239 return bench( 240 shellpath, 241 filepath, 242 options.warmup_runs, 243 options.counted_runs, 244 prepare, 245 func, 246 json.dumps(funcOpt), 247 stfu=options.stfu, 248 ) 249 250 bench_map = parsemark(gen_filepaths(dirpath), benchfile, options.stfu) 251 if options.baseline_path: 252 compare_bench.compare_immediate(bench_map, options.baseline_path) 253 return 0 254 255 256 if __name__ == "__main__": 257 sys.exit(main())