commit 34719179f66fddee9250238d79425aea3b48979e
parent 6913250961b28911054968a46136e35a0a7305d3
Author: Alex Franchuk <afranchuk@mozilla.com>
Date: Tue, 7 Oct 2025 13:48:13 +0000
Bug 1973820 p1 - Run a separate browser instance per GTest suite r=jmaher
This changes `./mach gtest` to, by default, run a separate process per
test suite. It adds the `--combine-suites` flag to use the old behavior.
If `--jobs` is specified, at most that many processes will be running at
one time.
Running a separate process per test suite ensures that tests are more
repeatable and don't affect each other. It also interestingly results in
a faster runtime than running all tests from one process, even when run
serially (perhaps due to memory allocator behavior?).
Differential Revision: https://phabricator.services.mozilla.com/D254995
Diffstat:
2 files changed, 241 insertions(+), 14 deletions(-)
diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py
@@ -21,6 +21,7 @@ from os import path
from pathlib import Path
import mozpack.path as mozpath
+from gtest.suites import get_gtest_suites, suite_filters
from mach.decorators import (
Command,
CommandArgument,
@@ -879,6 +880,13 @@ def join_ensure_dir(dir1, dir2):
help="Run the tests in parallel using multiple processes.",
)
@CommandArgument(
+ "--combine-suites",
+ action="store_true",
+ default=False,
+ help="Run multiple test suites in the same process invocation (as opposed "
+ "to the default behavior of running one process per test suite).",
+)
+@CommandArgument(
"--tbpl-parser",
"-t",
action="store_true",
@@ -982,6 +990,7 @@ def gtest(
command_context,
shuffle,
jobs,
+ combine_suites,
gtest_filter,
list_tests,
tbpl_parser,
@@ -1026,6 +1035,10 @@ def gtest(
if conditions.is_android(command_context):
if jobs != 1:
print("--jobs is not supported on Android and will be ignored")
+ if combine_suites:
+ print(
+ "--combine-suites is always the behavior on Android and will be ignored"
+ )
if enable_inc_origin_init:
print(
"--enable-inc-origin-init is not supported on Android and will"
@@ -1132,7 +1145,11 @@ def gtest(
gtest_filter_sets.list()
return 1
- if jobs == 1:
+ # Don't bother with multiple processes if:
+ # - listing tests
+ # - running the debugger
+ # - combining suites with one job
+ if list_tests or debug or (combine_suites and jobs == 1):
return command_context.run_process(
args=args,
append_env=gtest_env,
@@ -1145,26 +1162,104 @@ def gtest(
from mozprocess import ProcessHandlerMixin
- def handle_line(job_id, line):
- # Prepend the jobId
- line = "[%d] %s" % (job_id + 1, line.strip())
- command_context.log(logging.INFO, "GTest", {"line": line}, "{line}")
+ processes = []
- gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs)
- processes = {}
- for i in range(0, jobs):
- gtest_env["GTEST_SHARD_INDEX"] = str(i)
- processes[i] = ProcessHandlerMixin(
+ def add_process(job_id, env, **kwargs):
+ def log_line(line):
+ # Prepend the job identifier to output
+ command_context.log(
+ logging.INFO,
+ "GTest",
+ {"job_id": job_id, "line": line.strip()},
+ "[{job_id}] {line}",
+ )
+
+ proc = ProcessHandlerMixin(
[app_path, "-unittest"],
cwd=cwd,
- env=gtest_env,
- processOutputLine=[functools.partial(handle_line, i)],
universal_newlines=True,
+ env=env,
+ processOutputLine=log_line,
+ **kwargs,
)
- processes[i].run()
+ processes.append(proc)
+ return proc
+
+ if combine_suites:
+ # Use GTest sharding to create `jobs` processes
+ gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs)
+
+ for i in range(0, jobs):
+ env = gtest_env.copy()
+ env["GTEST_SHARD_INDEX"] = str(i)
+ add_process(str(i), env).run()
+ else:
+ # Make one process per test suite
+ suites = get_gtest_suites(args, cwd, gtest_env)
+
+ from threading import Event, Lock
+
+ processes_to_run = []
+ all_processes_run = Event()
+ running_suites = set()
+ process_state_lock = Lock()
+
+ def run_next(finished_suite=None):
+ """
+ Run another test suite process.
+
+ If `finished_suite` is provided, it will be considered as finished.
+ This updates the `running_suites` set and will signal the
+ `all_processes_run` Event when there are no longer any test suites
+ to start (though some may still be running).
+
+ This may be safely called from different threads
+ (ProcessHandlerMixin callbacks occur from separate threads).
+ """
+ # The changes here must be synchronized, so acquire a lock for the
+ # duration of the function.
+ with process_state_lock:
+ if finished_suite is not None:
+ running_suites.remove(finished_suite)
+ if len(processes_to_run) > 0:
+ next_suite, proc = processes_to_run.pop()
+ proc.run()
+ running_suites.add(next_suite)
+ command_context.log(
+ logging.DEBUG,
+ "GTest",
+ {},
+ f"Starting {next_suite} tests. {len(processes_to_run)} suites remain.",
+ )
+ else:
+ all_processes_run.set()
+ if len(running_suites) > 0:
+ command_context.log(
+ logging.INFO,
+ "GTest",
+ {},
+ f"Currently running suites: {', '.join(running_suites)}",
+ )
+
+ for filt in suite_filters(suites):
+ proc = add_process(
+ filt.suite,
+ filt(gtest_env),
+ onFinish=functools.partial(run_next, filt.suite),
+ )
+ processes_to_run.append((filt.suite, proc))
+
+ # Start a number of processes according to 'jobs'. As they finish,
+ # they'll each kick off another one.
+ for _ in range(jobs):
+ run_next()
+
+ # Wait for all processes to have been started, then wait on completion.
+ all_processes_run.wait()
+ # Wait on processes and return a non-zero exit code if any process does so.
exit_code = 0
- for process in processes.values():
+ for process in processes:
status = process.wait()
if status:
exit_code = status
diff --git a/testing/gtest/suites.py b/testing/gtest/suites.py
@@ -0,0 +1,132 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import itertools
+import os
+import re
+import subprocess
+
+
+def get_gtest_suites(args, cwd, gtest_env):
+ """
+ Get a list of gtest suite names from a gtest program.
+
+ * args - The arguments (including executable) for the gtest program.
+ * cwd - The working directory to use.
+ * gtest_env - Additional environment variables to set.
+
+ Returns a list of the suite names.
+ """
+ # List the tests to get the suite names
+ args.append("--gtest_list_tests")
+
+ env = {}
+ env.update(os.environ)
+ env.update(gtest_env)
+ completed_proc = subprocess.run(
+ args, cwd=cwd, env=env, capture_output=True, check=True, text=True
+ )
+ output = completed_proc.stdout
+
+ # Suite names are exclusively text without whitespace, and followed by
+ # a '.', optionally with ` #` and type parameter information. This is
+ # specific enough to reasonably filter out some extra strings output by
+ # firefox.
+ SUITE_REGEX = re.compile(r"(\S+).( # .*)?")
+
+ def get_suite_name(line):
+ match = SUITE_REGEX.fullmatch(line)
+ if match:
+ return match[1]
+
+ suites = list(
+ filter(lambda x: x is not None, map(get_suite_name, output.splitlines()))
+ )
+
+ # Remove the `--gtest_list_tests` arg that we added
+ args.pop()
+
+ return suites
+
+
+class _JoinedSubsetOfStrings:
+ """
+ Efficient creation of joined strings for subsets of a list of strings.
+
+ This allows creation of joined strings in O(1) instead of O(n) each time (n = list
+ length), with a one-time O(n) cost.
+ """
+
+ def __init__(self, between, strs):
+ """
+ Arguments:
+ * between - the string with which to join the strings
+ * strs - an iterable of strings
+ """
+ strs = list(strs)
+ self._string = between.join(strs)
+ betweenlen = len(between)
+ self._offsets = list(
+ itertools.accumulate(map(lambda s: len(s) + betweenlen, strs), initial=0)
+ )
+
+ def without(self, index):
+ """Create a joined string excluding the given index."""
+ return (
+ self._string[: self._offsets[index]]
+ + self._string[self._offsets[index + 1] :]
+ )
+
+
+class SuiteFilter:
+ def __init__(self, joined, index, suite):
+ self._joined = joined
+ self.index = index
+ self.suite = suite
+
+ def create(self, existing_filter=None):
+ """Create a filter to only run this suite."""
+ if existing_filter is None or existing_filter == "*":
+ return f"{self.suite}.*"
+ else:
+ return (
+ existing_filter
+ + (":" if "-" in existing_filter else "-")
+ + self._joined.without(self.index)
+ )
+
+ def set_in_env(self, env):
+ """
+ Set the filter to only run this suite in an environment mapping.
+
+ Returns the passed env.
+ """
+ env["GTEST_FILTER"] = self.create(env.get("GTEST_FILTER"))
+ return env
+
+ def __call__(self, val):
+ """
+ If called on a dict, creates a copy and forwards to `set_in_env`,
+ otherwise forwards to `create`.
+ """
+ if isinstance(val, dict):
+ return self.set_in_env(val.copy())
+ else:
+ return self.create(val)
+
+
+def suite_filters(suites):
+ """
+ Form gtest filters to limit tests to a single suite.
+
+ This is a generator that yields a SuiteFilter for each suite.
+
+ Arguments:
+ * suites - an iterable of the suite names
+ """
+ suites = list(suites)
+ joined = _JoinedSubsetOfStrings(":", map(lambda s: f"{s}.*", suites))
+
+ for i, suite in enumerate(suites):
+ yield SuiteFilter(joined, i, suite)
