commit 3c554294d0a55494120aa57d095f69d85e18e5b7
parent 298ad0fff1969cf27584f5e4665def81d90b05e8
Author: Alex Hochheiden <ahochheiden@mozilla.com>
Date: Thu, 13 Nov 2025 22:08:40 +0000
Bug 1704891 - Move checking for/setting up `node` executables to `nodeutil.py` r=Standard8,firefox-build-system-reviewers,perftest-reviewers,mozperftest-reviewers,sparky,glandium
Differential Revision: https://phabricator.services.mozilla.com/D264253
Diffstat:
11 files changed, 202 insertions(+), 263 deletions(-)
diff --git a/python/mozbuild/mozbuild/nodeutil.py b/python/mozbuild/mozbuild/nodeutil.py
@@ -5,14 +5,45 @@
import os
import platform
import subprocess
+import sys
from mozboot.util import get_tools_dir
from mozfile import which
+from mozfile.mozfile import remove as mozfileremove
from packaging.version import Version
NODE_MIN_VERSION = Version("12.22.12")
NPM_MIN_VERSION = Version("6.14.16")
+NODE_MACHING_VERSION_NOT_FOUND_MESSAGE = """
+Could not find Node.js executable later than %s.
+
+Executing `mach bootstrap --no-system-changes` should
+install a compatible version in ~/.mozbuild on most platforms.
+""".strip()
+
+NPM_MACHING_VERSION_NOT_FOUND_MESSAGE = """
+Could not find npm executable later than %s.
+
+Executing `mach bootstrap --no-system-changes` should
+install a compatible version in ~/.mozbuild on most platforms.
+""".strip()
+
+NODE_NOT_FOUND_MESSAGE = """
+nodejs is either not installed or is installed to a non-standard path.
+
+Executing `mach bootstrap --no-system-changes` should
+install a compatible version in ~/.mozbuild on most platforms.
+""".strip()
+
+NPM_NOT_FOUND_MESSAGE = """
+Node Package Manager (npm) is either not installed or installed to a
+non-standard path.
+
+Executing `mach bootstrap --no-system-changes` should
+install a compatible version in ~/.mozbuild on most platforms.
+""".strip()
+
def find_node_paths():
"""Determines the possible paths for node executables.
@@ -123,3 +154,146 @@ def find_executable(name, min_version, use_node_for_version_check=False):
return None, None
return exe, version.release
+
+
+def check_node_executables_valid():
+ node_path, version = find_node_executable()
+ if not node_path:
+ print(NODE_NOT_FOUND_MESSAGE)
+ return False
+ if not version:
+ print(NODE_MACHING_VERSION_NOT_FOUND_MESSAGE % NODE_MIN_VERSION)
+ return False
+
+ npm_path, version = find_npm_executable()
+ if not npm_path:
+ print(NPM_NOT_FOUND_MESSAGE)
+ return False
+ if not version:
+ print(NPM_MACHING_VERSION_NOT_FOUND_MESSAGE % NPM_MIN_VERSION)
+ return False
+
+ return True
+
+
+def package_setup(
+ package_root,
+ package_name,
+ should_update=False,
+ should_clobber=False,
+ no_optional=False,
+ skip_logging=False,
+):
+ """Ensure `package_name` at `package_root` is installed.
+
+ When `should_update` is true, clobber, install, and produce a new
+ "package-lock.json" file.
+
+ This populates `package_root/node_modules`.
+
+ """
+ orig_cwd = os.getcwd()
+
+ if should_update:
+ should_clobber = True
+
+ try:
+ # npm sometimes fails to respect cwd when it is run using check_call so
+ # we manually switch folders here instead.
+ os.chdir(package_root)
+
+ if should_clobber:
+ remove_directory(os.path.join(package_root, "node_modules"), skip_logging)
+
+ npm_path, _ = find_npm_executable()
+ if not npm_path:
+ return 1
+
+ node_path, _ = find_node_executable()
+ if not node_path:
+ return 1
+
+ extra_parameters = ["--loglevel=error"]
+
+ if no_optional:
+ extra_parameters.append("--no-optional")
+
+ package_lock_json_path = os.path.join(package_root, "package-lock.json")
+
+ if should_update:
+ cmd = [npm_path, "install"]
+ mozfileremove(package_lock_json_path)
+ else:
+ cmd = [npm_path, "ci"]
+
+ # On non-Windows, ensure npm is called via node, as node may not be in the
+ # path.
+ if platform.system() != "Windows":
+ cmd.insert(0, node_path)
+
+ cmd.extend(extra_parameters)
+
+ # Ensure that bare `node` and `npm` in scripts, including post-install scripts, finds the
+ # binary we're invoking with. Without this, it's easy for compiled extensions to get
+ # mismatched versions of the Node.js extension API.
+ path = os.environ.get("PATH", "").split(os.pathsep)
+ node_dir = os.path.dirname(node_path)
+ if node_dir not in path:
+ path = [node_dir] + path
+
+ if not skip_logging:
+ print(f'Installing {package_name} for mach using "{" ".join(cmd)}"...')
+ result = call_process(
+ package_name, cmd, append_env={"PATH": os.pathsep.join(path)}
+ )
+
+ if not result:
+ return 1
+
+ bin_path = os.path.join(package_root, "node_modules", ".bin", package_name)
+
+ if not skip_logging:
+ print(f"\n{package_name} installed successfully!")
+ print(f"\nNOTE: Your local {package_name} binary is at {bin_path}\n")
+
+ finally:
+ os.chdir(orig_cwd)
+
+
+def remove_directory(path, skip_logging=False):
+ if not skip_logging:
+ print(f"Clobbering {path}...")
+ if sys.platform.startswith("win") and have_winrm():
+ process = subprocess.Popen(["winrm", "-rf", path])
+ process.wait()
+ else:
+ mozfileremove(path)
+
+
+def call_process(name, cmd, cwd=None, append_env={}):
+ env = dict(os.environ)
+ env.update(append_env)
+
+ try:
+ with open(os.devnull, "w") as fnull:
+ subprocess.check_call(cmd, cwd=cwd, stdout=fnull, env=env)
+ except subprocess.CalledProcessError:
+ if cwd:
+ print(f"\nError installing {name} in the {cwd} folder, aborting.")
+ else:
+ print(f"\nError installing {name}, aborting.")
+
+ return False
+
+ return True
+
+
+def have_winrm():
+ # `winrm -h` should print 'winrm version ...' and exit 1
+ try:
+ p = subprocess.Popen(
+ ["winrm.exe", "-h"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+ )
+ return p.wait() == 1 and p.stdout.read().startswith("winrm")
+ except Exception:
+ return False
diff --git a/python/mozperftest/mozperftest/test/browsertime/runner.py b/python/mozperftest/mozperftest/test/browsertime/runner.py
@@ -7,7 +7,6 @@ import os
import pathlib
import re
import shutil
-import sys
from pathlib import Path
from mozperftest.test.browsertime.visualtools import get_dependencies, xvfb
@@ -94,20 +93,9 @@ class BrowsertimeRunner(NodeRunner):
self.virtualenv_manager = mach_cmd.virtualenv_manager
self._created_dirs = []
self._test_script = None
- self._setup_helper = None
self.get_binary_path = mach_cmd.get_binary_path
@property
- def setup_helper(self):
- if self._setup_helper is not None:
- return self._setup_helper
- sys.path.append(str(Path(self.topsrcdir, "tools", "lint", "eslint")))
- import setup_helper
-
- self._setup_helper = setup_helper
- return self._setup_helper
-
- @property
def artifact_cache_path(self):
"""Downloaded artifacts will be kept here."""
# The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
@@ -252,7 +240,9 @@ class BrowsertimeRunner(NodeRunner):
def _setup_node_packages(self, package_json_path):
# Install the browsertime Node.js requirements.
- if not self.setup_helper.check_node_executables_valid():
+ from mozbuild.nodeutil import check_node_executables_valid, package_setup
+
+ if not check_node_executables_valid():
return
should_clobber = self.get_arg("clobber")
@@ -271,7 +261,7 @@ class BrowsertimeRunner(NodeRunner):
)
install_url = self.get_arg("install-url")
- self.setup_helper.package_setup(
+ package_setup(
str(self.state_path),
"browsertime",
should_update=install_url is not None,
diff --git a/python/mozperftest/mozperftest/test/noderunner.py b/python/mozperftest/mozperftest/test/noderunner.py
@@ -2,7 +2,6 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
-import sys
import mozpack.path as mozpath
@@ -62,14 +61,13 @@ class NodeRunner(Layer):
def verify_node_install(self):
# check if Node is installed
- sys.path.append(mozpath.join(self.topsrcdir, "tools", "lint", "eslint"))
- import setup_helper
+ from mozbuild.nodeutil import check_node_executables_valid
with silence():
- node_valid = setup_helper.check_node_executables_valid()
+ node_valid = check_node_executables_valid()
if not node_valid:
# running again to get details printed out
- setup_helper.check_node_executables_valid()
+ check_node_executables_valid()
raise ValueError("Can't find Node. did you run ./mach bootstrap ?")
return True
diff --git a/python/mozperftest/mozperftest/tests/test_browsertime.py b/python/mozperftest/mozperftest/tests/test_browsertime.py
@@ -232,13 +232,6 @@ def test_browser_desktop(*mocked):
try:
with sys as s, browser as b, silence():
- # just checking that the setup_helper property gets
- # correctly initialized
- browsertime = browser.layers[-1]
- assert browsertime.setup_helper is not None
- helper = browsertime.setup_helper
- assert browsertime.setup_helper is helper
-
b(s(metadata))
finally:
shutil.rmtree(mach_cmd._mach_context.state_dir)
@@ -268,13 +261,6 @@ def test_existing_results(*mocked):
try:
with sys as s, browser as b, silence():
- # just checking that the setup_helper property gets
- # correctly initialized
- browsertime = browser.layers[-1]
- assert browsertime.setup_helper is not None
- helper = browsertime.setup_helper
- assert browsertime.setup_helper is helper
-
m = b(s(metadata))
results = m.get_results()
assert len(results) == 1
@@ -300,7 +286,9 @@ def test_add_options():
"mozperftest.test.noderunner.NodeRunner.verify_node_install", new=lambda x: True
)
@mock.patch("mozbuild.artifact_cache.ArtifactCache.fetch", new=fetch)
-@mock.patch("mozperftest.test.browsertime.runner.BrowsertimeRunner.setup_helper")
+@mock.patch(
+ "mozperftest.test.browsertime.runner.BrowsertimeRunner._setup_node_packages"
+)
def test_install_url(*mocked):
url = "https://here/tarball/" + "".join(
[random.choice(string.hexdigits[:-6]) for c in range(40)]
diff --git a/tools/browsertime/mach_commands.py b/tools/browsertime/mach_commands.py
@@ -280,8 +280,7 @@ def setup_browsertime(
):
r"""Install browsertime and visualmetrics.py prerequisites and the Node.js package."""
- sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint"))
- import setup_helper
+ from mozbuild.nodeutil import check_node_executables_valid, package_setup
if not new_upstream_url:
setup_prerequisites(command_context)
@@ -319,7 +318,7 @@ def setup_browsertime(
f.write(updated_body)
# Install the browsertime Node.js requirements.
- if not setup_helper.check_node_executables_valid():
+ if not check_node_executables_valid():
return 1
# To use a custom `geckodriver`, set
@@ -353,7 +352,7 @@ def setup_browsertime(
if IS_APPLE_SILICON and node_dir not in os.environ["PATH"]:
os.environ["PATH"] += os.pathsep + node_dir
- status = setup_helper.package_setup(
+ status = package_setup(
BROWSERTIME_ROOT,
"browsertime",
should_update=new_upstream_url != "",
@@ -581,11 +580,10 @@ def extra_default_args(command_context, args=[]):
def _verify_node_install(command_context):
# check if Node is installed
- sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint"))
- import setup_helper
+ from mozbuild.nodeutil import check_node_executables_valid
with silence():
- node_valid = setup_helper.check_node_executables_valid()
+ node_valid = check_node_executables_valid()
if not node_valid:
print("Can't find Node. did you run ./mach bootstrap ?")
return False
diff --git a/tools/lint/eslint/__init__.py b/tools/lint/eslint/__init__.py
@@ -11,7 +11,7 @@ import subprocess
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), "eslint"))
-from mozbuild.nodeutil import find_node_executable
+from mozbuild.nodeutil import check_node_executables_valid, find_node_executable
from mozlint import result
from eslint import prettier_utils, setup_helper
@@ -36,7 +36,7 @@ and try again.
def setup(root, **lintargs):
setup_helper.set_project_root(root)
- if not setup_helper.check_node_executables_valid():
+ if not check_node_executables_valid():
return 1
return setup_helper.eslint_maybe_setup()
diff --git a/tools/lint/eslint/setup_helper.py b/tools/lint/eslint/setup_helper.py
@@ -6,51 +6,15 @@
import json
import os
-import platform
import re
-import subprocess
-import sys
from filecmp import dircmp
from mozbuild.nodeutil import (
- NODE_MIN_VERSION,
- NPM_MIN_VERSION,
- find_node_executable,
- find_npm_executable,
+ package_setup,
+ remove_directory,
)
-from mozfile.mozfile import remove as mozfileremove
from packaging.version import Version
-NODE_MACHING_VERSION_NOT_FOUND_MESSAGE = """
-Could not find Node.js executable later than %s.
-
-Executing `mach bootstrap --no-system-changes` should
-install a compatible version in ~/.mozbuild on most platforms.
-""".strip()
-
-NPM_MACHING_VERSION_NOT_FOUND_MESSAGE = """
-Could not find npm executable later than %s.
-
-Executing `mach bootstrap --no-system-changes` should
-install a compatible version in ~/.mozbuild on most platforms.
-""".strip()
-
-NODE_NOT_FOUND_MESSAGE = """
-nodejs is either not installed or is installed to a non-standard path.
-
-Executing `mach bootstrap --no-system-changes` should
-install a compatible version in ~/.mozbuild on most platforms.
-""".strip()
-
-NPM_NOT_FOUND_MESSAGE = """
-Node Package Manager (npm) is either not installed or installed to a
-non-standard path.
-
-Executing `mach bootstrap --no-system-changes` should
-install a compatible version in ~/.mozbuild on most platforms.
-""".strip()
-
-
VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$")
CARET_VERSION_RANGE_RE = re.compile(r"^\^((\d+)\.\d+\.\d+)$")
@@ -85,129 +49,12 @@ def eslint_setup(package_root, package_name, should_clobber=False):
os.path.join(get_eslint_module_path(), "eslint-plugin-mozilla", "node_modules")
)
- package_setup(package_root, package_name, should_clobber=should_clobber)
-
-
-def remove_directory(path, skip_logging=False):
- if not skip_logging:
- print("Clobbering %s..." % path)
- if sys.platform.startswith("win") and have_winrm():
- process = subprocess.Popen(["winrm", "-rf", path])
- process.wait()
- else:
- mozfileremove(path)
-
-
-def package_setup(
- package_root,
- package_name,
- should_update=False,
- should_clobber=False,
- no_optional=False,
- skip_logging=False,
-):
- """Ensure `package_name` at `package_root` is installed.
-
- When `should_update` is true, clobber, install, and produce a new
- "package-lock.json" file.
-
- This populates `package_root/node_modules`.
-
- """
orig_project_root = get_project_root()
- orig_cwd = os.getcwd()
-
- if should_update:
- should_clobber = True
-
try:
set_project_root(package_root)
- sys.path.append(os.path.dirname(__file__))
-
- # npm sometimes fails to respect cwd when it is run using check_call so
- # we manually switch folders here instead.
- project_root = get_project_root()
- os.chdir(project_root)
-
- if should_clobber:
- remove_directory(os.path.join(project_root, "node_modules"), skip_logging)
-
- npm_path, _ = find_npm_executable()
- if not npm_path:
- return 1
-
- node_path, _ = find_node_executable()
- if not node_path:
- return 1
-
- extra_parameters = ["--loglevel=error"]
-
- if no_optional:
- extra_parameters.append("--no-optional")
-
- package_lock_json_path = os.path.join(get_project_root(), "package-lock.json")
-
- if should_update:
- cmd = [npm_path, "install"]
- mozfileremove(package_lock_json_path)
- else:
- cmd = [npm_path, "ci"]
-
- # On non-Windows, ensure npm is called via node, as node may not be in the
- # path.
- if platform.system() != "Windows":
- cmd.insert(0, node_path)
-
- cmd.extend(extra_parameters)
-
- # Ensure that bare `node` and `npm` in scripts, including post-install scripts, finds the
- # binary we're invoking with. Without this, it's easy for compiled extensions to get
- # mismatched versions of the Node.js extension API.
- path = os.environ.get("PATH", "").split(os.pathsep)
- node_dir = os.path.dirname(node_path)
- if node_dir not in path:
- path = [node_dir] + path
-
- if not skip_logging:
- print(
- 'Installing %s for mach using "%s"...' % (package_name, " ".join(cmd))
- )
- result = call_process(
- package_name, cmd, append_env={"PATH": os.pathsep.join(path)}
- )
-
- if not result:
- return 1
-
- bin_path = os.path.join(
- get_project_root(), "node_modules", ".bin", package_name
- )
-
- if not skip_logging:
- print("\n%s installed successfully!" % package_name)
- print("\nNOTE: Your local %s binary is at %s\n" % (package_name, bin_path))
-
+ package_setup(package_root, package_name, should_clobber=should_clobber)
finally:
set_project_root(orig_project_root)
- os.chdir(orig_cwd)
-
-
-def call_process(name, cmd, cwd=None, append_env={}):
- env = dict(os.environ)
- env.update(append_env)
-
- try:
- with open(os.devnull, "w") as fnull:
- subprocess.check_call(cmd, cwd=cwd, stdout=fnull, env=env)
- except subprocess.CalledProcessError:
- if cwd:
- print("\nError installing %s in the %s folder, aborting." % (name, cwd))
- else:
- print("\nError installing %s, aborting." % name)
-
- return False
-
- return True
def expected_installed_modules(package_root, package_name):
@@ -338,33 +185,6 @@ def version_in_range(version, version_range):
return False
-def get_possible_node_paths_win():
- """
- Return possible nodejs paths on Windows.
- """
- if platform.system() != "Windows":
- return []
-
- return list(
- {
- "%s\\nodejs" % os.environ.get("SystemDrive"),
- os.path.join(os.environ.get("ProgramFiles"), "nodejs"),
- os.path.join(os.environ.get("PROGRAMW6432"), "nodejs"),
- os.path.join(os.environ.get("PROGRAMFILES"), "nodejs"),
- }
- )
-
-
-def get_version(path):
- try:
- version_str = subprocess.check_output(
- [path, "--version"], stderr=subprocess.STDOUT, universal_newlines=True
- )
- return version_str
- except (subprocess.CalledProcessError, OSError):
- return None
-
-
def set_project_root(root=None):
"""Sets the project root to the supplied path, or works out what the root
is based on looking for 'mach'.
@@ -406,34 +226,3 @@ def get_project_root():
def get_eslint_module_path():
return os.path.join(get_project_root(), "tools", "lint", "eslint")
-
-
-def check_node_executables_valid():
- node_path, version = find_node_executable()
- if not node_path:
- print(NODE_NOT_FOUND_MESSAGE)
- return False
- if not version:
- print(NODE_MACHING_VERSION_NOT_FOUND_MESSAGE % NODE_MIN_VERSION)
- return False
-
- npm_path, version = find_npm_executable()
- if not npm_path:
- print(NPM_NOT_FOUND_MESSAGE)
- return False
- if not version:
- print(NPM_MACHING_VERSION_NOT_FOUND_MESSAGE % NPM_MIN_VERSION)
- return False
-
- return True
-
-
-def have_winrm():
- # `winrm -h` should print 'winrm version ...' and exit 1
- try:
- p = subprocess.Popen(
- ["winrm.exe", "-h"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
- )
- return p.wait() == 1 and p.stdout.read().startswith("winrm")
- except Exception:
- return False
diff --git a/tools/lint/node-licenses/__init__.py b/tools/lint/node-licenses/__init__.py
@@ -10,7 +10,7 @@ import sys
sys.path.append(os.path.join(os.path.dirname(__file__), "eslint"))
from eslint import setup_helper
-from mozbuild.nodeutil import find_node_executable
+from mozbuild.nodeutil import check_node_executables_valid, find_node_executable
from mozlint import result
from mozlint.pathutils import expand_exclusions
@@ -36,7 +36,7 @@ and try again.
def setup(root, **lintargs):
setup_helper.set_project_root(root)
- if not setup_helper.check_node_executables_valid():
+ if not check_node_executables_valid():
return 1
return setup_helper.eslint_maybe_setup()
diff --git a/tools/lint/stylelint/__init__.py b/tools/lint/stylelint/__init__.py
@@ -13,7 +13,7 @@ import sys
sys.path.append(os.path.join(os.path.dirname(__file__), "eslint"))
from eslint import prettier_utils, setup_helper
-from mozbuild.nodeutil import find_node_executable
+from mozbuild.nodeutil import check_node_executables_valid, find_node_executable
from mozlint import result
STYLELINT_ERROR_MESSAGE = """
@@ -38,7 +38,7 @@ FILE_EXT_REGEX = re.compile(r"\.[a-z0-9_]{2,10}$", re.IGNORECASE)
def setup(root, **lintargs):
setup_helper.set_project_root(root)
- if not setup_helper.check_node_executables_valid():
+ if not check_node_executables_valid():
return 1
return setup_helper.eslint_maybe_setup()
diff --git a/tools/moztreedocs/mach_commands.py b/tools/moztreedocs/mach_commands.py
@@ -123,10 +123,11 @@ def build_docs(
):
# TODO: Bug 1704891 - move the ESLint setup tools to a shared place.
import setup_helper
+ from mozbuild.nodeutil import check_node_executables_valid
setup_helper.set_project_root(command_context.topsrcdir)
- if not setup_helper.check_node_executables_valid():
+ if not check_node_executables_valid():
return 1
setup_helper.eslint_maybe_setup()
diff --git a/tools/ts/mach_commands.py b/tools/ts/mach_commands.py
@@ -174,8 +174,9 @@ def node(ctx, script, *args):
def maybe_setup(ctx):
sys.path.append(mozpath.join(ctx.topsrcdir, "tools", "lint", "eslint"))
import setup_helper
+ from mozbuild.nodeutil import check_node_executables_valid
- if not setup_helper.check_node_executables_valid():
+ if not check_node_executables_valid():
return 1
setup_helper.eslint_maybe_setup(package_name="TypeScript")
