tor-browser

json_results.py (7141B)

---

# Copyright 2014 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


import collections
import itertools
import json
import logging
import time

from pylib.base import base_test_result

def GenerateResultsDict(test_run_results, global_tags=None):
 """Create a results dict from |test_run_results| suitable for writing to JSON.
 Args:
   test_run_results: a list of base_test_result.TestRunResults objects.
 Returns:
   A results dict that mirrors the one generated by
     base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON.
 """
 # Example json output.
 # {
 #   "global_tags": [],
 #   "all_tests": [
 #     "test1",
 #     "test2",
 #    ],
 #   "disabled_tests": [],
 #   "per_iteration_data": [
 #     {
 #       "test1": [
 #         {
 #           "status": "SUCCESS",
 #           "elapsed_time_ms": 1,
 #           "output_snippet": "",
 #           "output_snippet_base64": "",
 #           "losless_snippet": "",
 #         },
 #         ...
 #       ],
 #       "test2": [
 #         {
 #           "status": "FAILURE",
 #           "elapsed_time_ms": 12,
 #           "output_snippet": "",
 #           "output_snippet_base64": "",
 #           "losless_snippet": "",
 #         },
 #         ...
 #       ],
 #     },
 #     {
 #       "test1": [
 #         {
 #           "status": "SUCCESS",
 #           "elapsed_time_ms": 1,
 #           "output_snippet": "",
 #           "output_snippet_base64": "",
 #           "losless_snippet": "",
 #         },
 #       ],
 #       "test2": [
 #         {
 #           "status": "FAILURE",
 #           "elapsed_time_ms": 12,
 #           "output_snippet": "",
 #           "output_snippet_base64": "",
 #           "losless_snippet": "",
 #         },
 #       ],
 #     },
 #     ...
 #   ],
 # }

 all_tests = set()
 per_iteration_data = []
 test_run_links = {}

 for test_run_result in test_run_results:
   iteration_data = collections.defaultdict(list)
   if isinstance(test_run_result, list):
     results_iterable = itertools.chain(*(t.GetAll() for t in test_run_result))
     for tr in test_run_result:
       test_run_links.update(tr.GetLinks())

   else:
     results_iterable = test_run_result.GetAll()
     test_run_links.update(test_run_result.GetLinks())

   for r in results_iterable:
     result_dict = {
         'status': r.GetType(),
         'elapsed_time_ms': r.GetDuration(),
         'output_snippet': r.GetLog(),
         'losless_snippet': True,
         'output_snippet_base64': '',
         'links': r.GetLinks(),
     }
     iteration_data[r.GetName()].append(result_dict)

   all_tests = all_tests.union(set(iteration_data.keys()))
   per_iteration_data.append(iteration_data)

 return {
   'global_tags': global_tags or [],
   'all_tests': sorted(list(all_tests)),
   # TODO(jbudorick): Add support for disabled tests within base_test_result.
   'disabled_tests': [],
   'per_iteration_data': per_iteration_data,
   'links': test_run_links,
 }


def GenerateJsonTestResultFormatDict(test_run_results, interrupted):
 """Create a results dict from |test_run_results| suitable for writing to JSON.

 Args:
   test_run_results: a list of base_test_result.TestRunResults objects.
   interrupted: True if tests were interrupted, e.g. timeout listing tests
 Returns:
   A results dict that mirrors the standard JSON Test Results Format.
 """

 tests = {}
 counts = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'CRASH': 0, 'TIMEOUT': 0}

 for test_run_result in test_run_results:
   if isinstance(test_run_result, list):
     results_iterable = itertools.chain(*(t.GetAll() for t in test_run_result))
   else:
     results_iterable = test_run_result.GetAll()

   for r in results_iterable:
     element = tests
     for key in r.GetName().split('.'):
       if key not in element:
         element[key] = {}
       element = element[key]

     element['expected'] = 'PASS'

     if r.GetType() == base_test_result.ResultType.PASS:
       result = 'PASS'
     elif r.GetType() == base_test_result.ResultType.SKIP:
       result = 'SKIP'
     elif r.GetType() == base_test_result.ResultType.CRASH:
       result = 'CRASH'
     elif r.GetType() == base_test_result.ResultType.TIMEOUT:
       result = 'TIMEOUT'
     else:
       result = 'FAIL'

     if 'actual' in element:
       element['actual'] += ' ' + result
     else:
       counts[result] += 1
       element['actual'] = result
       if result == 'FAIL':
         element['is_unexpected'] = True

     if r.GetDuration() != 0:
       element['time'] = r.GetDuration()

 # Fill in required fields.
 return {
     'interrupted': interrupted,
     'num_failures_by_type': counts,
     'path_delimiter': '.',
     'seconds_since_epoch': time.time(),
     'tests': tests,
     'version': 3,
 }


def GenerateJsonResultsFile(test_run_result, file_path, global_tags=None,
                           **kwargs):
 """Write |test_run_result| to JSON.

 This emulates the format of the JSON emitted by
 base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON.

 Args:
   test_run_result: a base_test_result.TestRunResults object.
   file_path: The path to the JSON file to write.
 """
 with open(file_path, 'w') as json_result_file:
   json_result_file.write(json.dumps(
       GenerateResultsDict(test_run_result, global_tags=global_tags),
       **kwargs))
   logging.info('Generated json results file at %s', file_path)


def GenerateJsonTestResultFormatFile(test_run_result, interrupted, file_path,
                                    **kwargs):
 """Write |test_run_result| to JSON.

 This uses the official Chromium Test Results Format.

 Args:
   test_run_result: a base_test_result.TestRunResults object.
   interrupted: True if tests were interrupted, e.g. timeout listing tests
   file_path: The path to the JSON file to write.
 """
 with open(file_path, 'w') as json_result_file:
   json_result_file.write(
       json.dumps(
           GenerateJsonTestResultFormatDict(test_run_result, interrupted),
           **kwargs))
   logging.info('Generated json results file at %s', file_path)


def ParseResultsFromJson(json_results):
 """Creates a list of BaseTestResult objects from JSON.

 Args:
   json_results: A JSON dict in the format created by
                 GenerateJsonResultsFile.
 """

 def string_as_status(s):
   if s in base_test_result.ResultType.GetTypes():
     return s
   return base_test_result.ResultType.UNKNOWN

 results_list = []
 testsuite_runs = json_results['per_iteration_data']
 for testsuite_run in testsuite_runs:
   for test, test_runs in testsuite_run.items():
     results_list.extend(
         [base_test_result.BaseTestResult(test,
                                          string_as_status(tr['status']),
                                          duration=tr['elapsed_time_ms'],
                                          log=tr.get('output_snippet'))
         for tr in test_runs])
 return results_list