tor-browser

perf_tests_results_helper.py (6579B)

---

# Copyright 2013 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


import re
import sys

import json
import logging
import math

from lib.common import perf_result_data_type


# Mapping from result type to test output
RESULT_TYPES = {perf_result_data_type.UNIMPORTANT: 'RESULT ',
               perf_result_data_type.DEFAULT: '*RESULT ',
               perf_result_data_type.INFORMATIONAL: '',
               perf_result_data_type.UNIMPORTANT_HISTOGRAM: 'HISTOGRAM ',
               perf_result_data_type.HISTOGRAM: '*HISTOGRAM '}


def _EscapePerfResult(s):
 """Escapes |s| for use in a perf result."""
 return re.sub('[\:|=/#&,]', '_', s)


def FlattenList(values):
 """Returns a simple list without sub-lists."""
 ret = []
 for entry in values:
   if isinstance(entry, list):
     ret.extend(FlattenList(entry))
   else:
     ret.append(entry)
 return ret


def GeomMeanAndStdDevFromHistogram(histogram_json):
 histogram = json.loads(histogram_json)
 # Handle empty histograms gracefully.
 if not 'buckets' in histogram:
   return 0.0, 0.0
 count = 0
 sum_of_logs = 0
 for bucket in histogram['buckets']:
   if 'high' in bucket:
     bucket['mean'] = (bucket['low'] + bucket['high']) / 2.0
   else:
     bucket['mean'] = bucket['low']
   if bucket['mean'] > 0:
     sum_of_logs += math.log(bucket['mean']) * bucket['count']
     count += bucket['count']

 if count == 0:
   return 0.0, 0.0

 sum_of_squares = 0
 geom_mean = math.exp(sum_of_logs / count)
 for bucket in histogram['buckets']:
   if bucket['mean'] > 0:
     sum_of_squares += (bucket['mean'] - geom_mean) ** 2 * bucket['count']
 return geom_mean, math.sqrt(sum_of_squares / count)


def _ValueToString(v):
 # Special case for floats so we don't print using scientific notation.
 if isinstance(v, float):
   return '%f' % v
 else:
   return str(v)


def _MeanAndStdDevFromList(values):
 avg = None
 sd = None
 if len(values) > 1:
   try:
     value = '[%s]' % ','.join([_ValueToString(v) for v in values])
     avg = sum([float(v) for v in values]) / len(values)
     sqdiffs = [(float(v) - avg) ** 2 for v in values]
     variance = sum(sqdiffs) / (len(values) - 1)
     sd = math.sqrt(variance)
   except ValueError:
     value = ', '.join(values)
 else:
   value = values[0]
 return value, avg, sd


def PrintPages(page_list):
 """Prints list of pages to stdout in the format required by perf tests."""
 print('Pages: [%s]' % ','.join([_EscapePerfResult(p) for p in page_list]))


def PrintPerfResult(measurement, trace, values, units,
                   result_type=perf_result_data_type.DEFAULT,
                   print_to_stdout=True):
 """Prints numerical data to stdout in the format required by perf tests.

 The string args may be empty but they must not contain any colons (:) or
 equals signs (=).

 Args:
   measurement: A description of the quantity being measured, e.g. "vm_peak".
       On the dashboard, this maps to a particular graph. Mandatory.
   trace: A description of the particular data point, e.g. "reference".
       On the dashboard, this maps to a particular "line" in the graph.
       Mandatory.
   values: A list of numeric measured values. An N-dimensional list will be
       flattened and treated as a simple list.
   units: A description of the units of measure, e.g. "bytes".
   result_type: Accepts values of perf_result_data_type.ALL_TYPES.
   print_to_stdout: If True, prints the output in stdout instead of returning
       the output to caller.

   Returns:
     String of the formated perf result.
 """
 assert perf_result_data_type.IsValidType(result_type), \
        'result type: %s is invalid' % result_type

 trace_name = _EscapePerfResult(trace)

 if (result_type == perf_result_data_type.UNIMPORTANT or
     result_type == perf_result_data_type.DEFAULT or
     result_type == perf_result_data_type.INFORMATIONAL):
   assert isinstance(values, list)
   assert '/' not in measurement
   flattened_values = FlattenList(values)
   assert len(flattened_values)
   value, avg, sd = _MeanAndStdDevFromList(flattened_values)
   output = '%s%s: %s%s%s %s' % (
       RESULT_TYPES[result_type],
       _EscapePerfResult(measurement),
       trace_name,
       # Do not show equal sign if the trace is empty. Usually it happens when
       # measurement is enough clear to describe the result.
       '= ' if trace_name else '',
       value,
       units)
 else:
   assert perf_result_data_type.IsHistogram(result_type)
   assert isinstance(values, list)
   # The histograms can only be printed individually, there's no computation
   # across different histograms.
   assert len(values) == 1
   value = values[0]
   output = '%s%s: %s= %s %s' % (
       RESULT_TYPES[result_type],
       _EscapePerfResult(measurement),
       trace_name,
       value,
       units)
   avg, sd = GeomMeanAndStdDevFromHistogram(value)

 if avg:
   output += '\nAvg %s: %f%s' % (measurement, avg, units)
 if sd:
   output += '\nSd  %s: %f%s' % (measurement, sd, units)
 if print_to_stdout:
   print(output)
   sys.stdout.flush()
 return output


def ReportPerfResult(chart_data, graph_title, trace_title, value, units,
                    improvement_direction='down', important=True):
 """Outputs test results in correct format.

 If chart_data is None, it outputs data in old format. If chart_data is a
 dictionary, formats in chartjson format. If any other format defaults to
 old format.

 Args:
   chart_data: A dictionary corresponding to perf results in the chartjson
       format.
   graph_title: A string containing the name of the chart to add the result
       to.
   trace_title: A string containing the name of the trace within the chart
       to add the result to.
   value: The value of the result being reported.
   units: The units of the value being reported.
   improvement_direction: A string denoting whether higher or lower is
       better for the result. Either 'up' or 'down'.
   important: A boolean denoting whether the result is important or not.
 """
 if chart_data and isinstance(chart_data, dict):
   chart_data['charts'].setdefault(graph_title, {})
   chart_data['charts'][graph_title][trace_title] = {
       'type': 'scalar',
       'value': value,
       'units': units,
       'improvement_direction': improvement_direction,
       'important': important
   }
 else:
   PrintPerfResult(graph_title, trace_title, [value], units)