compare_bench.py (2563B)
1 #!/usr/bin/env python2.4 2 """usage: %progname candidate_path baseline_path""" 3 4 import json 5 import optparse 6 from operator import itemgetter 7 8 9 def avg(seq): 10 return sum(seq) / len(seq) 11 12 13 def compare(current, baseline): 14 percent_speedups = [] 15 for key, current_result in iter(current.items()): 16 try: 17 baseline_result = baseline[key] 18 except KeyError: 19 print(key, "missing from baseline") 20 continue 21 val_getter = itemgetter("average_ms", "stddev_ms") 22 base_avg, base_stddev = val_getter(baseline_result) 23 current_avg, current_stddev = val_getter(current_result) 24 t_best = current_avg - current_stddev 25 t_worst = current_avg + current_stddev 26 base_t_best = base_avg - base_stddev 27 base_t_worst = base_avg + base_stddev 28 if t_worst < base_t_best: 29 # Worst takes less time (better) than baseline's best. 30 speedup = -((t_worst - base_t_best) / base_t_best) * 100 31 result = f"faster: {t_worst:6.2f}ms < baseline {base_t_best:6.2f}ms ({speedup:+6.2f}%)" 32 percent_speedups.append(speedup) 33 elif t_best > base_t_worst: 34 # Best takes more time (worse) than baseline's worst. 35 slowdown = -((t_best - base_t_worst) / base_t_worst) * 100 36 result = f"SLOWER: {t_best:6.2f}ms > baseline {base_t_worst:6.2f}ms ({slowdown:+6.2f}%) " 37 percent_speedups.append(slowdown) 38 else: 39 result = "Meh." 40 print(f"{key:30s}: {result}") 41 if percent_speedups: 42 print(f"Average speedup: {avg(percent_speedups):.2f}%") 43 44 45 def compare_immediate(current_map, baseline_path): 46 baseline_file = open(baseline_path) 47 baseline_map = json.load(baseline_file) 48 baseline_file.close() 49 compare(current_map, baseline_map) 50 51 52 def main(candidate_path, baseline_path): 53 candidate_file, baseline_file = open(candidate_path), open(baseline_path) 54 candidate = json.load(candidate_file) 55 baseline = json.load(baseline_file) 56 compare(candidate, baseline) 57 candidate_file.close() 58 baseline_file.close() 59 60 61 if __name__ == "__main__": 62 parser = optparse.OptionParser(usage=__doc__.strip()) 63 options, args = parser.parse_args() 64 try: 65 candidate_path = args.pop(0) 66 except IndexError: 67 parser.error("A JSON filepath to compare against baseline is required") 68 try: 69 baseline_path = args.pop(0) 70 except IndexError: 71 parser.error("A JSON filepath for baseline is required") 72 main(candidate_path, baseline_path)