[ tor-browser ].git.dasho

run-tests.py (14188B)
      1 #!/usr/bin/env python
      2 # This Source Code Form is subject to the terms of the Mozilla Public
      3 # License, v. 2.0. If a copy of the MPL was not distributed with this
      4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
      5 
      6 # run-tests.py -- Python harness for GDB SpiderMonkey support
      7 
      8 import os
      9 import re
     10 import subprocess
     11 import sys
     12 
     13 # From this directory:
     14 import progressbar
     15 from taskpool import TaskPool, get_cpu_count
     16 
     17 
     18 def _relpath(path, start=None):
     19    # Backported from Python 3.1 posixpath.py
     20    """Return a relative version of a path"""
     21 
     22    if not path:
     23        raise ValueError("no path specified")
     24 
     25    if start is None:
     26        start = os.curdir
     27 
     28    start_list = os.path.abspath(start).split(os.sep)
     29    path_list = os.path.abspath(path).split(os.sep)
     30 
     31    # Work out how much of the filepath is shared by start and path.
     32    i = len(os.path.commonprefix([start_list, path_list]))
     33 
     34    rel_list = [os.pardir] * (len(start_list) - i) + path_list[i:]
     35    if not rel_list:
     36        return os.curdir
     37    return os.path.join(*rel_list)
     38 
     39 
     40 os.path.relpath = _relpath
     41 
     42 # Characters that need to be escaped when used in shell words.
     43 shell_need_escapes = re.compile("[^\\w\\d%+,-./:=@'\"]", re.DOTALL)
     44 # Characters that need to be escaped within double-quoted strings.
     45 shell_dquote_escapes = re.compile('[^\\w\\d%+,-./:=@"]', re.DOTALL)
     46 
     47 
     48 def make_shell_cmd(l):
     49    def quote(s):
     50        if shell_need_escapes.search(s):
     51            if s.find("'") < 0:
     52                return "'" + s + "'"
     53            return '"' + shell_dquote_escapes.sub("\\g<0>", s) + '"'
     54        return s
     55 
     56    return " ".join([quote(_) for _ in l])
     57 
     58 
     59 # An instance of this class collects the lists of passing, failing, and
     60 # timing-out tests, runs the progress bar, and prints a summary at the end.
     61 class Summary:
     62    class SummaryBar(progressbar.ProgressBar):
     63        def __init__(self, limit):
     64            super().__init__("", limit, 24)
     65 
     66        def start(self):
     67            self.label = "[starting           ]"
     68            self.update(0)
     69 
     70        def counts(self, run, failures, timeouts):
     71            self.label = "[%4d|%4d|%4d|%4d]" % (run - failures, failures, timeouts, run)
     72            self.update(run)
     73 
     74    def __init__(self, num_tests):
     75        self.run = 0
     76        self.failures = []  # kind of judgemental; "unexpecteds"?
     77        self.timeouts = []
     78        if not OPTIONS.hide_progress:
     79            self.bar = Summary.SummaryBar(num_tests)
     80 
     81    # Progress bar control.
     82    def start(self):
     83        if not OPTIONS.hide_progress:
     84            self.bar.start()
     85 
     86    def update(self):
     87        if not OPTIONS.hide_progress:
     88            self.bar.counts(self.run, len(self.failures), len(self.timeouts))
     89 
     90    # Call 'thunk' to show some output, while getting the progress bar out of the way.
     91 
     92    def interleave_output(self, thunk):
     93        if not OPTIONS.hide_progress:
     94            self.bar.clear()
     95        thunk()
     96        self.update()
     97 
     98    def passed(self, test):
     99        self.run += 1
    100        self.update()
    101 
    102    def failed(self, test):
    103        self.run += 1
    104        self.failures.append(test)
    105        self.update()
    106 
    107    def timeout(self, test):
    108        self.run += 1
    109        self.timeouts.append(test)
    110        self.update()
    111 
    112    def finish(self):
    113        if not OPTIONS.hide_progress:
    114            self.bar.finish()
    115 
    116        if self.failures:
    117            print("tests failed:")
    118            for test in self.failures:
    119                test.show(sys.stdout)
    120 
    121            if OPTIONS.worklist:
    122                try:
    123                    with open(OPTIONS.worklist) as out:
    124                        for test in self.failures:
    125                            out.write(test.name + "\n")
    126                except OSError as err:
    127                    sys.stderr.write(
    128                        "Error writing worklist file '%s': %s" % (OPTIONS.worklist, err)
    129                    )
    130                    sys.exit(1)
    131 
    132            if OPTIONS.write_failures:
    133                try:
    134                    with open(OPTIONS.write_failures, "w") as out:
    135                        for test in self.failures:
    136                            test.show(out)
    137                except OSError as err:
    138                    sys.stderr.write(
    139                        "Error writing worklist file '%s': %s"
    140                        % (OPTIONS.write_failures, err)
    141                    )
    142                    sys.exit(1)
    143 
    144        if self.timeouts:
    145            print("tests timed out:")
    146            for test in self.timeouts:
    147                test.show(sys.stdout)
    148 
    149        if self.failures or self.timeouts:
    150            sys.exit(2)
    151 
    152 
    153 class Test(TaskPool.Task):
    154    def __init__(self, path, summary):
    155        super().__init__()
    156        self.test_path = path  # path to .py test file
    157        self.summary = summary
    158 
    159        # test.name is the name of the test relative to the top of the test
    160        # directory. This is what we use to report failures and timeouts,
    161        # and when writing test lists.
    162        self.name = os.path.relpath(self.test_path, OPTIONS.testdir)
    163 
    164        self.stdout = ""
    165        self.stderr = ""
    166        self.returncode = None
    167 
    168    def cmd(self):
    169        testlibdir = os.path.normpath(
    170            os.path.join(OPTIONS.testdir, "..", "lib-for-tests")
    171        )
    172        return [
    173            OPTIONS.gdb_executable,
    174            "-nw",  # Don't create a window (unnecessary?)
    175            "-nx",  # Don't read .gdbinit.
    176            "--ex",
    177            "add-auto-load-safe-path %s" % (OPTIONS.bindir,),
    178            "--ex",
    179            "set env LD_LIBRARY_PATH %s" % (OPTIONS.bindir,),
    180            "--ex",
    181            "file %s" % (os.path.join(OPTIONS.bindir, "gdb-tests"),),
    182            "--eval-command",
    183            "python testlibdir=%r" % (testlibdir,),
    184            "--eval-command",
    185            "python testscript=%r" % (self.test_path,),
    186            "--eval-command",
    187            "python exec(open(%r).read())" % os.path.join(testlibdir, "catcher.py"),
    188        ]
    189 
    190    def start(self, pipe, deadline):
    191        super().start(pipe, deadline)
    192        if OPTIONS.show_cmd:
    193            self.summary.interleave_output(lambda: self.show_cmd(sys.stdout))
    194 
    195    def onStdout(self, text):
    196        self.stdout += text
    197 
    198    def onStderr(self, text):
    199        self.stderr += text
    200 
    201    def onFinished(self, returncode):
    202        self.returncode = returncode
    203        if OPTIONS.show_output:
    204            self.summary.interleave_output(lambda: self.show_output(sys.stdout))
    205        if returncode != 0:
    206            self.summary.failed(self)
    207        else:
    208            self.summary.passed(self)
    209 
    210    def onTimeout(self):
    211        self.summary.timeout(self)
    212 
    213    def show_cmd(self, out):
    214        out.write("Command: %s\n" % (make_shell_cmd(self.cmd()),))
    215 
    216    def show_output(self, out):
    217        if self.stdout:
    218            out.write("Standard output:")
    219            out.write("\n" + self.stdout + "\n")
    220        if self.stderr:
    221            out.write("Standard error:")
    222            out.write("\n" + self.stderr + "\n")
    223 
    224    def show(self, out):
    225        out.write(self.name + "\n")
    226        if OPTIONS.write_failure_output:
    227            self.show_cmd(out)
    228            self.show_output(out)
    229            out.write("GDB exit code: %r\n" % (self.returncode,))
    230 
    231 
    232 def find_tests(dir, substring=None):
    233    ans = []
    234    for dirpath, _, filenames in os.walk(dir):
    235        if dirpath == ".":
    236            continue
    237        for filename in filenames:
    238            if not filename.endswith(".py"):
    239                continue
    240            test = os.path.join(dirpath, filename)
    241            if substring is None or substring in os.path.relpath(test, dir):
    242                ans.append(test)
    243    return ans
    244 
    245 
    246 def build_test_exec(builddir):
    247    subprocess.check_call(["make"], cwd=builddir)
    248 
    249 
    250 def run_tests(tests, summary):
    251    jobs = OPTIONS.workercount
    252    # python 3.3 fixed a bug with concurrently writing .pyc files.
    253    # https://bugs.python.org/issue13146
    254    embedded_version = (
    255        subprocess.check_output([
    256            OPTIONS.gdb_executable,
    257            "--batch",
    258            "--ex",
    259            "python import sys; print(sys.hexversion)",
    260        ])
    261        .decode("ascii")
    262        .strip()
    263    )
    264    if hex(int(embedded_version)) < "0x3030000":
    265        jobs = 1
    266 
    267    pool = TaskPool(tests, job_limit=jobs, timeout=OPTIONS.timeout)
    268    pool.run_all()
    269 
    270 
    271 OPTIONS = None
    272 
    273 
    274 def main(argv):
    275    global OPTIONS
    276    script_path = os.path.abspath(__file__)
    277    script_dir = os.path.dirname(script_path)
    278 
    279    # OBJDIR is a standalone SpiderMonkey build directory. This is where we
    280    # find the SpiderMonkey shared library to link against.
    281    #
    282    # The [TESTS] optional arguments are paths of test files relative
    283    # to the jit-test/tests directory.
    284    from optparse import OptionParser
    285 
    286    op = OptionParser(usage="%prog [options] OBJDIR [TESTS...]")
    287    op.add_option(
    288        "-s",
    289        "--show-cmd",
    290        dest="show_cmd",
    291        action="store_true",
    292        help="show GDB shell command run",
    293    )
    294    op.add_option(
    295        "-o",
    296        "--show-output",
    297        dest="show_output",
    298        action="store_true",
    299        help="show output from GDB",
    300    )
    301    op.add_option(
    302        "-x",
    303        "--exclude",
    304        dest="exclude",
    305        action="append",
    306        help="exclude given test dir or path",
    307    )
    308    op.add_option(
    309        "-t",
    310        "--timeout",
    311        dest="timeout",
    312        type=float,
    313        default=150.0,
    314        help="set test timeout in seconds",
    315    )
    316    op.add_option(
    317        "-j",
    318        "--worker-count",
    319        dest="workercount",
    320        type=int,
    321        help="Run [WORKERCOUNT] tests at a time",
    322    )
    323    op.add_option(
    324        "--no-progress",
    325        dest="hide_progress",
    326        action="store_true",
    327        help="hide progress bar",
    328    )
    329    op.add_option(
    330        "--worklist",
    331        dest="worklist",
    332        metavar="FILE",
    333        help="Read tests to run from [FILE] (or run all if [FILE] not found);\n"
    334        "write failures back to [FILE]",
    335    )
    336    op.add_option(
    337        "-r",
    338        "--read-tests",
    339        dest="read_tests",
    340        metavar="FILE",
    341        help="Run test files listed in [FILE]",
    342    )
    343    op.add_option(
    344        "-w",
    345        "--write-failures",
    346        dest="write_failures",
    347        metavar="FILE",
    348        help="Write failing tests to [FILE]",
    349    )
    350    op.add_option(
    351        "--write-failure-output",
    352        dest="write_failure_output",
    353        action="store_true",
    354        help="With --write-failures=FILE, additionally write the output of failed "
    355        "tests to [FILE]",
    356    )
    357    op.add_option(
    358        "--gdb",
    359        dest="gdb_executable",
    360        metavar="EXECUTABLE",
    361        default="gdb",
    362        help="Run tests with [EXECUTABLE], rather than plain 'gdb'.",
    363    )
    364    op.add_option(
    365        "--srcdir",
    366        dest="srcdir",
    367        default=os.path.abspath(os.path.join(script_dir, "..")),
    368        help="Use SpiderMonkey sources in [SRCDIR].",
    369    )
    370    op.add_option(
    371        "--testdir",
    372        dest="testdir",
    373        default=os.path.join(script_dir, "tests"),
    374        help="Find tests in [TESTDIR].",
    375    )
    376    op.add_option(
    377        "--builddir", dest="builddir", help="Build test executable from [BUILDDIR]."
    378    )
    379    op.add_option("--bindir", dest="bindir", help="Run test executable from [BINDIR].")
    380    (OPTIONS, args) = op.parse_args(argv)
    381    if len(args) < 1:
    382        op.error("missing OBJDIR argument")
    383    OPTIONS.objdir = os.path.abspath(args[0])
    384 
    385    test_args = args[1:]
    386 
    387    if not OPTIONS.workercount:
    388        OPTIONS.workercount = get_cpu_count()
    389 
    390    # Compute defaults for OPTIONS.builddir and OPTIONS.bindir now, since we've
    391    # computed OPTIONS.objdir.
    392    if not OPTIONS.builddir:
    393        OPTIONS.builddir = os.path.join(OPTIONS.objdir, "js", "src", "gdb")
    394    if not OPTIONS.bindir:
    395        OPTIONS.bindir = os.path.join(OPTIONS.objdir, "dist", "bin")
    396 
    397    test_set = set()
    398 
    399    # All the various sources of test names accumulate.
    400    if test_args:
    401        for arg in test_args:
    402            test_set.update(find_tests(OPTIONS.testdir, arg))
    403    if OPTIONS.worklist:
    404        try:
    405            with open(OPTIONS.worklist) as f:
    406                for line in f:
    407                    test_set.update(os.path.join(OPTIONS.testdir, line.strip("\n")))
    408        except OSError:
    409            # With worklist, a missing file means to start the process with
    410            # the complete list of tests.
    411            sys.stderr.write(
    412                "Couldn't read worklist file '%s'; running all tests\n"
    413                % (OPTIONS.worklist,)
    414            )
    415            test_set = set(find_tests(OPTIONS.testdir))
    416    if OPTIONS.read_tests:
    417        try:
    418            with open(OPTIONS.read_tests) as f:
    419                for line in f:
    420                    test_set.update(os.path.join(OPTIONS.testdir, line.strip("\n")))
    421        except OSError as err:
    422            sys.stderr.write(
    423                "Error trying to read test file '%s': %s\n" % (OPTIONS.read_tests, err)
    424            )
    425            sys.exit(1)
    426 
    427    # If none of the above options were passed, and no tests were listed
    428    # explicitly, use the complete set.
    429    if not test_args and not OPTIONS.worklist and not OPTIONS.read_tests:
    430        test_set = set(find_tests(OPTIONS.testdir))
    431 
    432    if OPTIONS.exclude:
    433        exclude_set = set()
    434        for exclude in OPTIONS.exclude:
    435            exclude_set.update(find_tests(OPTIONS.testdir, exclude))
    436        test_set -= exclude_set
    437 
    438    if not test_set:
    439        sys.stderr.write("No tests found matching command line arguments.\n")
    440        sys.exit(1)
    441 
    442    summary = Summary(len(test_set))
    443    test_list = [Test(_, summary) for _ in sorted(test_set)]
    444 
    445    # Build the test executable from all the .cpp files found in the test
    446    # directory tree.
    447    try:
    448        build_test_exec(OPTIONS.builddir)
    449    except subprocess.CalledProcessError as err:
    450        sys.stderr.write("Error building test executable: %s\n" % (err,))
    451        sys.exit(1)
    452 
    453    # Run the tests.
    454    try:
    455        summary.start()
    456        run_tests(test_list, summary)
    457        summary.finish()
    458    except OSError as err:
    459        sys.stderr.write("Error running tests: %s\n" % (err,))
    460        sys.exit(1)
    461 
    462    sys.exit(0)
    463 
    464 
    465 if __name__ == "__main__":
    466    main(sys.argv[1:])
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE