commit b9f589ffe2173b236735cc134ba1f6965f4121da
parent d749d6e28edf452e9bdba8a5da98f02bee8a6fb3
Author: Geoff Brown <gbrown@mozilla.com>
Date: Fri, 3 Oct 2025 18:01:30 +0000
Bug 1773091 - Archive desktop builds on autoland backstop pushes r=ahal,taskgraph-reviewers,releng-reviewers,bhearsum
Differential Revision: https://phabricator.services.mozilla.com/D218468
Diffstat:
6 files changed, 311 insertions(+), 3 deletions(-)
diff --git a/taskcluster/config.yml b/taskcluster/config.yml
@@ -601,9 +601,13 @@ workers:
implementation: beetmover
os: linux
worker-type:
- by-release-level:
- production: '{trust-domain}-3-beetmover'
- staging: '{trust-domain}-1-beetmover'
+ by-project:
+ # Autoland needs level 3 for integration beetmover tasks
+ autoland: '{trust-domain}-3-beetmover'
+ default:
+ by-release-level:
+ production: '{trust-domain}-3-beetmover'
+ staging: '{trust-domain}-1-beetmover'
beetmover-android:
provisioner: scriptworker-k8s
implementation: scriptworker-beetmover
diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst
@@ -270,6 +270,13 @@ Beetmover, takes specific artifacts, "Beets", and pushes them to a location outs
of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
process determines the final location and a "pretty" name (versioned product name)
+beetmover-integration
+---------------------
+Beetmover, takes specific artifacts, "Beets", and pushes them to a location outside
+of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
+process determines the final location and a "pretty" name (versioned product name)
+This separate kind archives builds from the autoland branch.
+
beetmover-l10n
--------------
diff --git a/taskcluster/gecko_taskgraph/manifests/firefox_integration.yml b/taskcluster/gecko_taskgraph/manifests/firefox_integration.yml
@@ -0,0 +1,148 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+# This file contains exhaustive information about all the release artifacs that
+# are needed within a type of release.
+#
+# Structure
+# --------
+# `s3_bucket_paths` -- prefix to be used per product to correctly access our S3 buckets
+# `default_locales` -- list of locales to be used when composing upstream artifacts or the list of
+# destinations. If given an empty locale, it uses these locales instead.
+# `tasktype_map` -- mapping between task reference and task type, particularly usefule when
+# composing the upstreamArtifacts for scriptworker.
+# `platform_names` -- various platform mappings used in reckoning artifacts or other paths
+# `default` -- a default entry, which the mappings extend and override in such a way that
+# final path full-destinations will be a concatenation of the following:
+# `s3_bucket_paths`, `destinations`, `locale_prefix`, `pretty_name`
+# `from` -- specifies the dependency(ies) from which to expect the particular artifact
+# `all_locales` -- boolean argument to specify whether that particular artifact is to be expected
+# for all locales or just the default one
+# `description` -- brief summary of what that artifact is
+# `locale_prefix` -- prefix to be used in the final destination paths, whether that's for default locale or not
+# `source_path_modifier` -- any parent dir that might be used in between artifact prefix and filename at source location
+# for example `public/build` vs `public/build/ach/`.
+# `destinations` -- final list of directories where to push the artifacts in S3
+# `pretty_name` -- the final name the artifact will have at destination
+# `checksums_path` -- the name to identify one artifact within the checksums file
+# `not_for_platforms` -- filtering option to avoid associating an artifact with a specific platform
+# `only_for_platforms` -- filtering option to exclusively include the association of an artifact for a specific platform
+# `partials_only` -- filtering option to avoid associating an artifact unless this flag is present
+# `update_balrog_manifest`-- flag needed downstream in beetmover jobs to reckon the balrog manifest
+# `from_buildid` -- flag needed downstream in beetmover jobs to reckon the balrog manifest
+
+s3_bucket_paths:
+ - pub/firefox/integration
+default_locales:
+ - en-US
+tasktype_map:
+ build: build
+ signing: signing
+ repackage: repackage
+platform_names:
+ filename_platform:
+ by-platform:
+ linux64: 'linux64'
+ linux64-aarch64-shippable: 'linux-aarch64'
+ linux64-asan-reporter-shippable: 'linux-x86_64-asan-reporter'
+ macosx64: 'mac'
+ macosx64-shippable: 'mac-shippable'
+ macosx64-aarch64-shippable: 'mac-aarch64'
+ win32: 'win32'
+ win32-shippable: 'win32-shippable'
+ win64: 'win64'
+ win64-shippable: 'win64-shippable'
+ win64-aarch64: 'win64-aarch64'
+ win64-aarch64-shippable: 'win64-aarch64'
+ win64-asan-reporter-shippable: 'win64-asan-reporter'
+
+default: &default
+ from:
+ - build
+ all_locales: false
+ description: "TO_BE_OVERRIDDEN"
+ locale_prefix: ''
+ source_path_modifier:
+ by-locale:
+ default: '${locale}'
+ en-US: ''
+ destinations:
+ - ${branch}/${year}/${month}/${day}/${head_rev}
+ expiry: "2 years"
+
+mapping:
+ target.json:
+ <<: *default
+ description: "Various compile and moz_app flags baked together in a json file"
+ pretty_name: firefox-${version}.${locale}.${filename_platform}.json
+ checksums_path: firefox-${version}.${locale}.${filename_platform}.json
+ target.mozinfo.json:
+ <<: *default
+ description: "Various compile and moz_app flags baked together in a json file"
+ pretty_name: firefox-${version}.${locale}.${filename_platform}.mozinfo.json
+ checksums_path: firefox-${version}.${locale}.${filename_platform}.mozinfo.json
+ target.txt:
+ <<: *default
+ description: "File containing buildid and revision"
+ pretty_name: firefox-${version}.${locale}.${filename_platform}.txt
+ checksums_path: firefox-${version}.${locale}.${filename_platform}.txt
+ target_info.txt:
+ <<: *default
+ description: "File containing the buildID"
+ pretty_name: firefox-${version}.${locale}.${filename_platform}_info.txt
+ checksums_path: firefox-${version}.${locale}.${filename_platform}_info.txt
+ target.jsshell.zip:
+ <<: *default
+ description: "Set of shells to allow test snippets of Javascript code without needing to reload the page"
+ pretty_name: jsshell-${filename_platform}.zip
+ checksums_path: jsshell-${filename_platform}.zip
+ target.tar.xz:
+ <<: *default
+ description: "Main installer for Linux platforms"
+ only_for_platforms:
+ - linux64
+ - linux64-shippable
+ - linux64-aarch64-shippable
+ - linux64-asan-reporter-shippable
+ pretty_name: firefox-${version}.${locale}.${filename_platform}.tar.xz
+ checksums_path: firefox-${version}.${locale}.${filename_platform}.tar.xz
+ target.dmg:
+ <<: *default
+ description: "Main package disk image for Mac OS X platforms"
+ # Attribution-l10n jobs don't have locale in the artifact path
+ source_path_modifier: ""
+ only_for_platforms:
+ - macosx64
+ - macosx64-shippable
+ - macosx64-aarch64-shippable
+ pretty_name: firefox-${version}.${locale}.${filename_platform}.dmg
+ checksums_path: firefox-${version}.${locale}.${filename_platform}.dmg
+ target.zip:
+ <<: *default
+ description: "Main package installer for Windows platforms"
+ only_for_platforms:
+ - win32
+ - win32-shippable
+ - win64
+ - win64-shippable
+ - win64-aarch64
+ - win64-aarch64-shippable
+ - win64-asan-reporter-shippable
+ pretty_name: firefox-${version}.${locale}.${filename_platform}.zip
+ checksums_path: firefox-${version}.${locale}.${filename_platform}.zip
+ target.installer.exe:
+ <<: *default
+ description: "Main installer for Windows platforms"
+ # Need to add the installer path since it's sourcing from build kind instead of repackage-signing
+ source_path_modifier: 'install/sea/'
+ only_for_platforms:
+ - win32
+ - win32-shippable
+ - win64
+ - win64-shippable
+ - win64-aarch64
+ - win64-aarch64-shippable
+ - win64-asan-reporter-shippable
+ pretty_name: firefox-${version}.${locale}.${filename_platform}.installer.exe
+ checksums_path: firefox-${version}.${locale}.${filename_platform}.installer.exe
diff --git a/taskcluster/gecko_taskgraph/transforms/beetmover_integration.py b/taskcluster/gecko_taskgraph/transforms/beetmover_integration.py
@@ -0,0 +1,102 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.dependencies import get_primary_dependency
+from taskgraph.util.keyed_by import evaluate_keyed_by
+from taskgraph.util.schema import Schema, optionally_keyed_by
+from voluptuous import Optional, Required
+
+from gecko_taskgraph.transforms.beetmover import craft_release_properties
+from gecko_taskgraph.transforms.task import task_description_schema
+from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
+from gecko_taskgraph.util.scriptworker import (
+ generate_beetmover_artifact_map,
+ generate_beetmover_upstream_artifacts,
+)
+
+beetmover_description_schema = Schema(
+ {
+ Required("label"): str,
+ Required("description"): str,
+ Required("dependencies"): task_description_schema["dependencies"],
+ Required("if-dependencies"): task_description_schema["if-dependencies"],
+ Optional("treeherder"): task_description_schema["treeherder"],
+ Required("run-on-projects"): task_description_schema["run-on-projects"],
+ Optional("attributes"): task_description_schema["attributes"],
+ Optional("task-from"): task_description_schema["task-from"],
+ Required("worker-type"): task_description_schema["worker-type"],
+ Required("scopes"): optionally_keyed_by("project", [str]),
+ }
+)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def remove_name(config, tasks):
+ for job in tasks:
+ if "name" in job:
+ del job["name"]
+ yield job
+
+
+transforms.add_validate(beetmover_description_schema)
+
+
+@transforms.add
+def make_task_description(config, tasks):
+ for task in tasks:
+ dep_task = get_primary_dependency(config, task)
+ assert dep_task
+
+ attributes = copy_attributes_from_dependent_job(dep_task)
+ attributes.update(task.get("attributes", {}))
+
+ treeherder = task.get("treeherder", {})
+ dep_th_platform = (
+ dep_task.task.get("extra", {})
+ .get("treeherder", {})
+ .get("machine", {})
+ .get("platform", "")
+ )
+ treeherder.setdefault("platform", f"{dep_th_platform}/opt")
+
+ task["description"] = task["description"].format(
+ build_platform=attributes.get("build_platform"),
+ build_type=attributes.get("build_type"),
+ )
+
+ task["scopes"] = evaluate_keyed_by(
+ task["scopes"],
+ "beetmover-integration",
+ {"project": config.params.get("project")},
+ )
+
+ if task.get("locale"):
+ attributes["locale"] = task["locale"]
+ task["attributes"] = attributes
+
+ yield task
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+ for job in jobs:
+ locale = job["attributes"].get("locale")
+ platform = job["attributes"]["build_platform"]
+ job["worker"] = {
+ "release-properties": craft_release_properties(config, job),
+ "upstream-artifacts": generate_beetmover_upstream_artifacts(
+ config, job, platform, locale
+ ),
+ "artifact-map": generate_beetmover_artifact_map(
+ config, job, platform=platform, locale=locale
+ ),
+ }
+ yield job
diff --git a/taskcluster/gecko_taskgraph/util/scriptworker.py b/taskcluster/gecko_taskgraph/util/scriptworker.py
@@ -751,7 +751,9 @@ def generate_beetmover_artifact_map(config, job, **kwargs):
"build_number": config.params["build_number"],
"year": upload_date.year,
"month": upload_date.strftime("%m"), # zero-pad the month
+ "day": upload_date.strftime("%d"),
"upload_date": upload_date.strftime("%Y-%m-%d-%H-%M-%S"),
+ "head_rev": config.params["head_rev"],
}
)
kwargs.update(**platforms)
diff --git a/taskcluster/kinds/beetmover-integration/kind.yml b/taskcluster/kinds/beetmover-integration/kind.yml
@@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+ - taskgraph.transforms.from_deps
+ - gecko_taskgraph.transforms.name_sanity
+ - gecko_taskgraph.transforms.beetmover_integration
+ - gecko_taskgraph.transforms.task
+
+kind-dependencies:
+ # Primary dependencies
+ - build
+
+only-for-build-platforms:
+ - linux64/opt
+ - macosx64/opt
+ - win32/opt
+ - win64/opt
+
+tasks:
+ beetmover-integration:
+ description: Beetmover integration submission - {build_platform}/{build_type}
+ from-deps:
+ group-by: platform
+ attributes:
+ artifact_map: taskcluster/gecko_taskgraph/manifests/firefox_integration.yml
+ if-dependencies:
+ - build
+ run-on-projects: [autoland]
+ treeherder:
+ tier: 2
+ kind: build
+ symbol: BM-int
+ worker-type: beetmover
+ scopes:
+ by-project:
+ autoland:
+ - project:releng:beetmover:bucket:release
+ - project:releng:beetmover:action:direct-push-to-bucket
+ default:
+ - project:releng:beetmover:bucket:dep
+ - project:releng:beetmover:action:direct-push-to-bucket
