commit 06c9e1d6039de1d2485c4f9051396a62d11c9397
parent f1c44ff0f01e6a1bd97ce456a650c4c5d4e0a562
Author: Andrew Halberstadt <ahal@mozilla.com>
Date: Fri, 31 Oct 2025 13:32:02 +0000
Bug 1996363 - Support both the Git and Mercurial 'run-task' scripts in debian base image, r=taskgraph-reviewers,bhearsum
We're going to be supporting cloning from both Github and hg.mozilla.org
throughout the migration. The `taskcluster/scripts/run-task` script only
supports Mercurial.
The upstream one in the vendor dir, supports both Git and Mercurial, but
is missing many features and optimization made to the Gecko copy.
Rather than try and merge the two scripts into one, my plan is to make
both of them available on the workers as `run-task-hg` and
`run-task-git` respectively. Transform logic will pick which one to use
based on the appropriate context.
Once the migration is finished and everything is cloning from Github,
we'll delete `taskcluster/scripts/run-task` from the tree. This avoids
the need to spend effort consolidating the two.
Differential Revision: https://phabricator.services.mozilla.com/D270344
Diffstat:
5 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/taskcluster/docker/debian-base/Dockerfile b/taskcluster/docker/debian-base/Dockerfile
@@ -61,7 +61,9 @@ COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
# %include taskcluster/scripts/run-task
-COPY topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task
+COPY topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task-hg
+# %include third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task
+COPY topsrcdir/third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task /builds/worker/bin/run-task-git
# %include third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content
ADD topsrcdir/third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content /builds/worker/bin/fetch-content
diff --git a/taskcluster/docker/fetch/Dockerfile b/taskcluster/docker/fetch/Dockerfile
@@ -27,7 +27,9 @@ RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAG
&& /usr/local/sbin/clean_packages.sh $DOCKER_IMAGE_PACKAGES
# %include taskcluster/scripts/run-task
-ADD topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task
+COPY topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task-hg
+# %include third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task
+COPY topsrcdir/third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task /builds/worker/bin/run-task-git
# %include third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content
ADD topsrcdir/third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content /builds/worker/bin/fetch-content
diff --git a/taskcluster/docker/ubuntu1804-base/Dockerfile b/taskcluster/docker/ubuntu1804-base/Dockerfile
@@ -62,7 +62,9 @@ COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
COPY topsrcdir/taskcluster/docker/recipes/dot-config /builds/worker/.config
# %include taskcluster/scripts/run-task
-COPY topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task
+COPY topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task-hg
+# %include third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task
+COPY topsrcdir/third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task /builds/worker/bin/run-task-git
# %include third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content
ADD topsrcdir/third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content /builds/worker/bin/fetch-content
diff --git a/taskcluster/gecko_taskgraph/transforms/job/run_task.py b/taskcluster/gecko_taskgraph/transforms/job/run_task.py
@@ -126,7 +126,10 @@ def script_url(config, script):
def docker_worker_run_task(config, job, taskdesc):
run = job["run"]
worker = taskdesc["worker"] = job["worker"]
- command = ["/builds/worker/bin/run-task"]
+ run_task_bin = (
+ "run-task-git" if config.params["repository_type"] == "git" else "run-task-hg"
+ )
+ command = [f"/builds/worker/bin/{run_task_bin}"]
common_setup(config, job, taskdesc, command)
if run["tooltool-downloads"]:
diff --git a/taskcluster/gecko_taskgraph/transforms/task.py b/taskcluster/gecko_taskgraph/transforms/task.py
@@ -316,6 +316,13 @@ def verify_index(config, index):
raise Exception(UNSUPPORTED_INDEX_PRODUCT_ERROR.format(product=product))
+RUN_TASK_RE = re.compile(r"run-task(-(git|hg))?$")
+
+
+def is_run_task(cmd: str) -> bool:
+ return bool(re.search(RUN_TASK_RE, cmd))
+
+
@payload_builder(
"docker-worker",
schema={
@@ -503,7 +510,7 @@ def build_docker_worker_payload(config, task, task_def):
if "max-run-time" in worker:
payload["maxRunTime"] = worker["max-run-time"]
- run_task = payload.get("command", [""])[0].endswith("run-task")
+ run_task = is_run_task(payload.get("command", [""])[0])
# run-task exits EXIT_PURGE_CACHES if there is a problem with caches.
# Automatically retry the tasks and purge caches if we see this exit
@@ -2564,7 +2571,7 @@ def check_run_task_caches(config, tasks):
command = payload.get("command") or [""]
main_command = command[0] if isinstance(command[0], str) else ""
- run_task = main_command.endswith("run-task")
+ run_task = is_run_task(main_command)
require_sparse_cache = False
have_sparse_cache = False
