tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

base_test_result.py (10388B)

      1 # Copyright 2013 The Chromium Authors
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 """Module containing base test results classes."""
      6 
      7 
      8 import functools
      9 import re
     10 import sys
     11 import threading
     12 
     13 from lib.results import result_types
     14 
     15 # This must match the source adding the suffix: bit.ly/3Zmwwyx
     16 MULTIPROCESS_SUFFIX = '__multiprocess_mode'
     17 
     18 # This must match the source adding the suffix: bit.ly/3Qt0Ww4
     19 _NULL_MUTATION_SUFFIX = '__null_'
     20 _MUTATION_SUFFIX_PATTERN = re.compile(r'^(.*)__([a-zA-Z]+)\.\.([a-zA-Z]+)_$')
     21 
     22 
     23 class ResultType:
     24  """Class enumerating test types.
     25 
     26  Wraps the results defined in //build/util/lib/results/.
     27  """
     28  PASS = result_types.PASS
     29  SKIP = result_types.SKIP
     30  FAIL = result_types.FAIL
     31  CRASH = result_types.CRASH
     32  TIMEOUT = result_types.TIMEOUT
     33  UNKNOWN = result_types.UNKNOWN
     34  NOTRUN = result_types.NOTRUN
     35 
     36  @staticmethod
     37  def GetTypes():
     38    """Get a list of all test types."""
     39    return [ResultType.PASS, ResultType.SKIP, ResultType.FAIL,
     40            ResultType.CRASH, ResultType.TIMEOUT, ResultType.UNKNOWN,
     41            ResultType.NOTRUN]
     42 
     43 
     44 @functools.total_ordering
     45 class BaseTestResult:
     46  """Base class for a single test result."""
     47 
     48  def __init__(self, name, test_type, duration=0, log='', failure_reason=None):
     49    """Construct a BaseTestResult.
     50 
     51    Args:
     52      name: Name of the test which defines uniqueness.
     53      test_type: Type of the test result as defined in ResultType.
     54      duration: Time it took for the test to run in milliseconds.
     55      log: An optional string listing any errors.
     56    """
     57    assert name
     58    assert test_type in ResultType.GetTypes()
     59    self._name = name
     60    self._test_type = test_type
     61    self._duration = duration
     62    self._log = log
     63    self._failure_reason = failure_reason
     64    self._links = {}
     65    self._webview_multiprocess_mode = MULTIPROCESS_SUFFIX in name
     66 
     67  def __str__(self):
     68    return self._name
     69 
     70  def __repr__(self):
     71    return self._name
     72 
     73  def __eq__(self, other):
     74    return self.GetName() == other.GetName()
     75 
     76  def __lt__(self, other):
     77    return self.GetName() < other.GetName()
     78 
     79  def __hash__(self):
     80    return hash(self._name)
     81 
     82  def SetName(self, name):
     83    """Set the test name.
     84 
     85    Because we're putting this into a set, this should only be used if moving
     86    this test result into another set.
     87    """
     88    self._name = name
     89 
     90  def GetName(self):
     91    """Get the test name."""
     92    return self._name
     93 
     94  def GetNameForResultSink(self):
     95    """Get the test name to be reported to resultsink."""
     96    raw_name = self.GetName()
     97 
     98    # The name can include suffixes encoding Webview variant data:
     99    # a Webview multiprocess mode suffix and an AwSettings mutation suffix.
    100    # If both are present, the mutation suffix will come after the multiprocess
    101    # suffix. The mutation suffix can either be "__null_" or "__{key}..{value}_"
    102    #
    103    # Examples:
    104    # (...)AwSettingsTest#testAssetUrl__multiprocess_mode__allMutations..true_
    105    # (...)AwSettingsTest#testAssetUrl__multiprocess_mode__null_
    106    # (...)AwSettingsTest#testAssetUrl__allMutations..true_
    107    # org.chromium.android_webview.test.AwSettingsTest#testAssetUrl__null_
    108 
    109    # first, strip any AwSettings mutation parameter information
    110    # from the RHS of the raw_name
    111    if raw_name.endswith(_NULL_MUTATION_SUFFIX):
    112      raw_name = raw_name[:-len(_NULL_MUTATION_SUFFIX)]
    113    elif match := _MUTATION_SUFFIX_PATTERN.search(raw_name):
    114      raw_name = match.group(1)
    115 
    116    # At this stage, the name will only have the multiprocess suffix appended,
    117    # if applicable.
    118    #
    119    # Examples:
    120    # (...)AwSettingsTest#testAssetUrl__multiprocess_mode
    121    # org.chromium.android_webview.test.AwSettingsTest#testAssetUrl
    122 
    123    # then check for multiprocess mode suffix and strip it, if present
    124    if self._webview_multiprocess_mode:
    125      assert raw_name.endswith(
    126          MULTIPROCESS_SUFFIX
    127      ), 'multiprocess mode test raw name should have the corresponding suffix'
    128      return raw_name[:-len(MULTIPROCESS_SUFFIX)]
    129    return raw_name
    130 
    131  def SetType(self, test_type):
    132    """Set the test result type."""
    133    assert test_type in ResultType.GetTypes()
    134    self._test_type = test_type
    135 
    136  def GetType(self):
    137    """Get the test result type."""
    138    return self._test_type
    139 
    140  def GetDuration(self):
    141    """Get the test duration."""
    142    return self._duration
    143 
    144  def SetLog(self, log):
    145    """Set the test log."""
    146    self._log = log
    147 
    148  def GetLog(self):
    149    """Get the test log."""
    150    return self._log
    151 
    152  def SetFailureReason(self, failure_reason):
    153    """Set the reason the test failed.
    154 
    155    This should be the first failure the test encounters and exclude any stack
    156    trace.
    157    """
    158    self._failure_reason = failure_reason
    159 
    160  def GetFailureReason(self):
    161    """Get the reason the test failed.
    162 
    163    Returns None if the test did not fail or if the reason the test failed is
    164    unknown.
    165    """
    166    return self._failure_reason
    167 
    168  def SetLink(self, name, link_url):
    169    """Set link with test result data."""
    170    self._links[name] = link_url
    171 
    172  def GetLinks(self):
    173    """Get dict containing links to test result data."""
    174    return self._links
    175 
    176  def GetVariantForResultSink(self):
    177    """Get the variant dict to be reported to result sink."""
    178    variants = {}
    179    if match := _MUTATION_SUFFIX_PATTERN.search(self.GetName()):
    180      # variant keys need to be lowercase
    181      variants[match.group(2).lower()] = match.group(3)
    182    if self._webview_multiprocess_mode:
    183      variants['webview_multiprocess_mode'] = 'Yes'
    184    return variants or None
    185 
    186 
    187 class TestRunResults:
    188  """Set of results for a test run."""
    189 
    190  def __init__(self):
    191    self._links = {}
    192    self._results = set()
    193    self._results_lock = threading.RLock()
    194 
    195  def SetLink(self, name, link_url):
    196    """Add link with test run results data."""
    197    self._links[name] = link_url
    198 
    199  def GetLinks(self):
    200    """Get dict containing links to test run result data."""
    201    return self._links
    202 
    203  def GetLogs(self):
    204    """Get the string representation of all test logs."""
    205    with self._results_lock:
    206      s = []
    207      for test_type in ResultType.GetTypes():
    208        if test_type != ResultType.PASS:
    209          for t in sorted(self._GetType(test_type)):
    210            log = t.GetLog()
    211            if log:
    212              s.append('[%s] %s:' % (test_type, t))
    213              s.append(log)
    214      if sys.version_info.major == 2:
    215        decoded = [u.decode(encoding='utf-8', errors='ignore') for u in s]
    216        return '\n'.join(decoded)
    217      return '\n'.join(s)
    218 
    219  def GetGtestForm(self):
    220    """Get the gtest string representation of this object."""
    221    with self._results_lock:
    222      s = []
    223      plural = lambda n, s, p: '%d %s' % (n, p if n != 1 else s)
    224      tests = lambda n: plural(n, 'test', 'tests')
    225 
    226      s.append('[==========] %s ran.' % (tests(len(self.GetAll()))))
    227      s.append('[  PASSED  ] %s.' % (tests(len(self.GetPass()))))
    228 
    229      skipped = self.GetSkip()
    230      if skipped:
    231        s.append('[  SKIPPED ] Skipped %s, listed below:' % tests(len(skipped)))
    232        for t in sorted(skipped):
    233          s.append('[  SKIPPED ] %s' % str(t))
    234 
    235      all_failures = self.GetFail().union(self.GetCrash(), self.GetTimeout(),
    236          self.GetUnknown())
    237      if all_failures:
    238        s.append('[  FAILED  ] %s, listed below:' % tests(len(all_failures)))
    239        for t in sorted(self.GetFail()):
    240          s.append('[  FAILED  ] %s' % str(t))
    241        for t in sorted(self.GetCrash()):
    242          s.append('[  FAILED  ] %s (CRASHED)' % str(t))
    243        for t in sorted(self.GetTimeout()):
    244          s.append('[  FAILED  ] %s (TIMEOUT)' % str(t))
    245        for t in sorted(self.GetUnknown()):
    246          s.append('[  FAILED  ] %s (UNKNOWN)' % str(t))
    247        s.append('')
    248        s.append(plural(len(all_failures), 'FAILED TEST', 'FAILED TESTS'))
    249      return '\n'.join(s)
    250 
    251  def GetShortForm(self):
    252    """Get the short string representation of this object."""
    253    with self._results_lock:
    254      s = []
    255      s.append('ALL: %d' % len(self._results))
    256      for test_type in ResultType.GetTypes():
    257        s.append('%s: %d' % (test_type, len(self._GetType(test_type))))
    258      return ''.join([x.ljust(15) for x in s])
    259 
    260  def __str__(self):
    261    return self.GetGtestForm()
    262 
    263  def AddResult(self, result):
    264    """Add |result| to the set.
    265 
    266    Args:
    267      result: An instance of BaseTestResult.
    268    """
    269    assert isinstance(result, BaseTestResult)
    270    with self._results_lock:
    271      self._results.discard(result)
    272      self._results.add(result)
    273 
    274  def AddResults(self, results):
    275    """Add |results| to the set.
    276 
    277    Args:
    278      results: An iterable of BaseTestResult objects.
    279    """
    280    with self._results_lock:
    281      for t in results:
    282        self.AddResult(t)
    283 
    284  def AddTestRunResults(self, results):
    285    """Add the set of test results from |results|.
    286 
    287    Args:
    288      results: An instance of TestRunResults.
    289    """
    290    assert isinstance(results, TestRunResults), (
    291           'Expected TestRunResult object: %s' % type(results))
    292    with self._results_lock:
    293      # pylint: disable=W0212
    294      self._results.update(results._results)
    295 
    296  def GetAll(self):
    297    """Get the set of all test results."""
    298    with self._results_lock:
    299      return self._results.copy()
    300 
    301  def _GetType(self, test_type):
    302    """Get the set of test results with the given test type."""
    303    with self._results_lock:
    304      return set(t for t in self._results if t.GetType() == test_type)
    305 
    306  def GetPass(self):
    307    """Get the set of all passed test results."""
    308    return self._GetType(ResultType.PASS)
    309 
    310  def GetSkip(self):
    311    """Get the set of all skipped test results."""
    312    return self._GetType(ResultType.SKIP)
    313 
    314  def GetFail(self):
    315    """Get the set of all failed test results."""
    316    return self._GetType(ResultType.FAIL)
    317 
    318  def GetCrash(self):
    319    """Get the set of all crashed test results."""
    320    return self._GetType(ResultType.CRASH)
    321 
    322  def GetTimeout(self):
    323    """Get the set of all timed out test results."""
    324    return self._GetType(ResultType.TIMEOUT)
    325 
    326  def GetUnknown(self):
    327    """Get the set of all unknown test results."""
    328    return self._GetType(ResultType.UNKNOWN)
    329 
    330  def GetNotPass(self):
    331    """Get the set of all non-passed test results."""
    332    return self.GetAll() - self.GetPass()
    333 
    334  def DidRunPass(self):
    335    """Return whether the test run was successful."""
    336    return not self.GetNotPass() - self.GetSkip()