commit 9c34d818ade0ce5c879a858133c1a3ca9246f7cd
parent 22284e85a1de490c37bb63364be4a3cb03c6657a
Author: Joel Maher <joel.maher@gmail.com>
Date: Fri, 21 Nov 2025 20:45:29 +0000
Bug 2001438 - run trainhop tests as variant instead of standalone suite. r=mconley,taskgraph-reviewers,bhearsum,home-newtab-reviewers
Differential Revision: https://phabricator.services.mozilla.com/D273493
Diffstat:
7 files changed, 117 insertions(+), 110 deletions(-)
diff --git a/browser/extensions/newtab/docs/v2-system-addon/train_hopping.md b/browser/extensions/newtab/docs/v2-system-addon/train_hopping.md
@@ -183,13 +183,13 @@ These tests are currently `Tier 2`, so failures do not automatically result in a
[This link](https://treeherder.mozilla.org/jobs?repo=mozilla-central&searchStr=trainhop) should show the most recent `trainhop` jobs occurring on `main`.
-The Treeherder "group name" for those jobs is `nt-trainhop`, and the symbol for the `mochitest-browser` tests are `Mbc-beta` and `Mbc-release` for the tests running on the most recent Beta and Release versions, respectively.
+The Treeherder "group name" for those jobs is `M-trainhop-rel` and `M-trainhop-beta`, and the symbol is `bc` for the tests running on the most recent Beta and Release versions, respectively.
As these are `mochitest-browser` tests, they suffer some of the same issues with intermittency that plague all `mochitest-browser` tests. The jobs can be retriggered in the event that a given failure appears to be intermittent, but we should all continue to strive to write more resilient and less non-deterministic tests when possible. It is righteous and useful to attempt to fix intermittent test failures for these jobs.
## Debugging automated tests
-If it comes to pass that the automated train-hop compatibility tests report a permanent failure on Beta and/or Release via the `Mbc-beta` or `Mbc-release` jobs, train-hop compatibility should be considered broken for that channel, and an investigation should begin immediately to determine what needs to be done to restore train-hop compatibility. No train-hops should be performed on the channel for which train-hop compatibility is broken.
+If it comes to pass that the automated train-hop compatibility tests report a permanent failure on Beta and/or Release via the `M-trainhop-*(bc)` jobs, train-hop compatibility should be considered broken for that channel, and an investigation should begin immediately to determine what needs to be done to restore train-hop compatibility. No train-hops should be performed on the channel for which train-hop compatibility is broken.
The first step is to examine the failure logs from automation to determine if anything immediately obvious jumps out as a “smoking gun”. Look for relevant error messages near the point where the test job failed. If a specific test failed, look for clues around when that test started and concluded. If no tests managed to run correctly, then the issue is likely to do with initializing the XPI (or in rare cases, [might be related to the test framework itself](https://bugzilla.mozilla.org/show_bug.cgi?id=1971180)).
diff --git a/taskcluster/gecko_taskgraph/transforms/test/variant.py b/taskcluster/gecko_taskgraph/transforms/test/variant.py
@@ -81,7 +81,9 @@ def split_variants(config, tasks):
def replace_task_items(task_key, variant_key):
for item in variant_key:
if isinstance(variant_key[item], dict):
- task_key[item] = replace_task_items(task_key[item], variant_key[item])
+ task_key[item] = replace_task_items(
+ task_key.get(item, {}), variant_key[item]
+ )
else:
task_key[item] = variant_key[item]
return task_key
diff --git a/taskcluster/kinds/mochitest/kind.yml b/taskcluster/kinds/mochitest/kind.yml
@@ -310,8 +310,26 @@ tasks:
- msix
- vertical-tabs
- inc-origin-init
+ - trainhop-beta
+ - trainhop-release
run-on-projects:
by-variant:
+ trainhop-beta:
+ by-test-platform:
+ .*-wayland.*: []
+ .*-32.*: []
+ linux2404-64(-shippable)?/opt: ['trunk']
+ macosx.*-64(-shippable)?(-qr)?/opt: ['trunk']
+ windows11-64-24h2(-shippable)?/opt: ['trunk']
+ default: []
+ trainhop-release:
+ by-test-platform:
+ .*-wayland.*: []
+ .*-32.*: []
+ linux2404-64(-shippable)?/opt: ['trunk']
+ macosx.*-64(-shippable)?(-qr)?/opt: ['trunk']
+ windows11-64-24h2(-shippable)?/opt: ['trunk']
+ default: []
webrender-sw:
by-test-platform:
linux.*64/debug: ['all']
@@ -354,6 +372,8 @@ tasks:
default: built-projects
tier:
by-variant:
+ trainhop-beta: 2
+ trainhop-release: 2
webrender-sw:
by-test-platform:
linux.*64-asan/opt: 1
@@ -362,6 +382,8 @@ tasks:
default: default
chunks:
by-variant:
+ trainhop-release: 1
+ trainhop-beta: 1
msix:
by-test-platform:
.*/debug: 16
@@ -1261,99 +1283,3 @@ tasks:
by-test-platform:
linux.*64-asan/opt: xlarge-noscratch # runs out of memory otherwise using 8GB
default: default
-
- mochitest-browser-newtab-trainhop-beta:
- description: Mochitest-browser for Newtab using latest Beta builds
- suite:
- name: mochitest-browser-chrome
- run-on-projects:
- by-test-platform:
- linux.*64(-shippable)?/.*: ['trunk']
- macosx.*64(-shippable)?(-qr)?/.*: ['trunk']
- windows1.*-64.*(-shippable)?/.*: ['trunk']
- default: []
- treeherder-symbol: nt-trainhop(Mbc-beta)
- tier: 2
- loopback-video: true
- test-manifest-loader: null # don't load tests in the taskgraph
- mozharness:
- mochitest-flavor: browser
- chunked: false
- extra-options:
- - --tag=newtab
- - --install-extension=newtab.xpi
- # This preference is normally set via Nimbus when doing a New Tab
- # train-hop, but when running the tests in automation like this, we
- # install the XPI using the more traditional extension install
- # mechanism.
- #
- # This is kind of a hack, but so long as this string here is not
- # empty, we bypass the uninstall that occurs at startup, and read
- # from the installed XPI.
- - --setpref=browser.newtabpage.trainhopAddon.version=any
- instance-size: default
- dependencies:
- newtab-xpi-job: build-extensions-browser
- fetches:
- newtab-xpi-job:
- - artifact: ../browser-extensions/newtab.xpi
- extract: false
- target:
- by-test-platform:
- linux.*:
- index: gecko.v2.mozilla-beta.shippable.latest.firefox.linux64-opt
- name: build/target.tar.xz
- macosx.*:
- index: gecko.v2.mozilla-beta.shippable.latest.firefox.macosx64-opt
- name: build/target.dmg
- windows.*:
- index: gecko.v2.mozilla-beta.shippable.latest.firefox.win64-opt
- name: build/target.zip
-
- mochitest-browser-newtab-trainhop-release:
- description: Mochitest-browser for Newtab using latest Release builds
- suite:
- name: mochitest-browser-chrome
- run-on-projects:
- by-test-platform:
- linux.*64(-shippable)?/.*: ['trunk']
- macosx.*64(-shippable)?(-qr)?/.*: ['trunk']
- windows1.*-64.*(-shippable)?/.*: ['trunk']
- default: []
- treeherder-symbol: nt-trainhop(Mbc-release)
- tier: 2
- loopback-video: true
- test-manifest-loader: null # don't load tests in the taskgraph
- mozharness:
- mochitest-flavor: browser
- chunked: false
- extra-options:
- - --tag=newtab
- - --install-extension=newtab.xpi
- # This preference is normally set via Nimbus when doing a New Tab
- # train-hop, but when running the tests in automation like this, we
- # install the XPI using the more traditional extension install
- # mechanism.
- #
- # This is kind of a hack, but so long as this string here is not
- # empty, we bypass the uninstall that occurs at startup, and read
- # from the installed XPI.
- - --setpref=browser.newtabpage.trainhopAddon.version=any
- instance-size: default
- dependencies:
- newtab-xpi-job: build-extensions-browser
- fetches:
- newtab-xpi-job:
- - artifact: ../browser-extensions/newtab.xpi
- extract: false
- target:
- by-test-platform:
- linux.*:
- index: gecko.v2.mozilla-release.shippable.latest.firefox.linux64-opt
- name: build/target.tar.xz
- macosx.*:
- index: gecko.v2.mozilla-release.shippable.latest.firefox.macosx64-opt
- name: build/target.dmg
- windows.*:
- index: gecko.v2.mozilla-release.shippable.latest.firefox.win64-opt
- name: build/target.zip
diff --git a/taskcluster/test_configs/test-platforms.yml b/taskcluster/test_configs/test-platforms.yml
@@ -65,7 +65,6 @@ linux2404-64/opt:
- linux-tests
- web-platform-tests
- web-platform-tests-backlog
- - newtab-trainhop-tests
linux2404-64-shippable/opt:
build-platform: linux64-shippable/opt
@@ -74,7 +73,6 @@ linux2404-64-shippable/opt:
- linux-tests
- web-platform-tests
- web-platform-tests-backlog
- - newtab-trainhop-tests
linux2404-64/debug:
build-platform: linux64/debug
@@ -246,7 +244,6 @@ windows11-64-24h2/opt:
- windows-talos
- browsertime
- windows-talos-webgl
- - newtab-trainhop-tests
windows11-64-24h2-shippable/opt:
build-platform: win64-shippable/opt
@@ -265,7 +262,6 @@ windows11-64-24h2-shippable/opt:
- windows-talos
- browsertime
- windows-talos-webgl
- - newtab-trainhop-tests
windows11-64-24h2-devedition/opt:
build-platform: win64-devedition/opt
@@ -340,7 +336,6 @@ macosx1470-64-shippable/opt:
- macosx1470-64-tests
- webgpu
- web-platform-tests-backlog
- - newtab-trainhop-tests
macosx1470-64-devedition/opt:
build-platform: macosx64-devedition/opt
@@ -355,7 +350,6 @@ macosx1470-64/opt:
- web-platform-tests-backlog
- webgpu
- browsertime
- - newtab-trainhop-tests
macosx1470-64/debug:
build-platform: macosx64/debug
diff --git a/taskcluster/test_configs/test-sets.yml b/taskcluster/test_configs/test-sets.yml
@@ -162,10 +162,6 @@ mochitest-valgrind:
desktop-screenshot-capture:
- mochitest-browser-screenshots
-newtab-trainhop-tests:
- - mochitest-browser-newtab-trainhop-beta
- - mochitest-browser-newtab-trainhop-release
-
### Specific platform test sets
# Linux
diff --git a/taskcluster/test_configs/variants.yml b/taskcluster/test_configs/variants.yml
@@ -89,6 +89,95 @@ aab:
replace:
target: "geckoview-test_runner.aab"
+trainhop-release:
+ description: Mochitest-browser for Newtab using latest Release builds
+ component: "Firefox::New Tab Page"
+ suffix: "trainhop-rel"
+ mozinfo: "trainhop"
+ expiration: None
+ merge:
+ worker:
+ env:
+ MOZHARNESS_TEST_TAG:
+ - "newtab"
+ mozharness:
+ chunked: false
+ extra-options:
+ - --install-extension=newtab.xpi
+ # This preference is normally set via Nimbus when doing a New Tab
+ # train-hop, but when running the tests in automation like this, we
+ # install the XPI using the more traditional extension install
+ # mechanism.
+ #
+ # This is kind of a hack, but so long as this string here is not
+ # empty, we bypass the uninstall that occurs at startup, and read
+ # from the installed XPI.
+ - --setpref=browser.newtabpage.trainhopAddon.version=any
+ dependencies:
+ newtab-xpi-job: build-extensions-browser
+ replace:
+ fetches:
+ newtab-xpi-job:
+ - artifact: ../browser-extensions/newtab.xpi
+ extract: false
+ target:
+ by-test-platform:
+ linux.*:
+ index: gecko.v2.mozilla-release.shippable.latest.firefox.linux64-opt
+ name: build/target.tar.xz
+ macosx.*:
+ index: gecko.v2.mozilla-release.shippable.latest.firefox.macosx64-opt
+ name: build/target.dmg
+ windows.*:
+ index: gecko.v2.mozilla-release.shippable.latest.firefox.win64-opt
+ name: build/target.zip
+
+
+trainhop-beta:
+ description: Mochitest-browser for Newtab using latest Beta builds
+ component: "Firefox::New Tab Page"
+ suffix: "trainhop-beta"
+ mozinfo: "trainhop"
+ expiration: None
+ merge:
+ worker:
+ env:
+ MOZHARNESS_TEST_TAG:
+ - "newtab"
+ mozharness:
+ chunked: false
+ extra-options:
+ - --install-extension=newtab.xpi
+ # This preference is normally set via Nimbus when doing a New Tab
+ # train-hop, but when running the tests in automation like this, we
+ # install the XPI using the more traditional extension install
+ # mechanism.
+ #
+ # This is kind of a hack, but so long as this string here is not
+ # empty, we bypass the uninstall that occurs at startup, and read
+ # from the installed XPI.
+ - --setpref=browser.newtabpage.trainhopAddon.version=any
+ dependencies:
+ newtab-xpi-job: build-extensions-browser
+
+ replace:
+ fetches:
+ newtab-xpi-job:
+ - artifact: ../browser-extensions/newtab.xpi
+ extract: false
+ target:
+ by-test-platform:
+ linux.*:
+ index: gecko.v2.mozilla-beta.shippable.latest.firefox.linux64-opt
+ name: build/target.tar.xz
+ macosx.*:
+ index: gecko.v2.mozilla-beta.shippable.latest.firefox.macosx64-opt
+ name: build/target.dmg
+ windows.*:
+ index: gecko.v2.mozilla-beta.shippable.latest.firefox.win64-opt
+ name: build/target.zip
+
+
geckoview-no-fission:
description: "{description} without fission enabled"
component: "GeckoView::General"
diff --git a/tools/tryselect/try_presets.yml b/tools/tryselect/try_presets.yml
@@ -107,7 +107,7 @@ desktop-newtab-trainhop:
selector: fuzzy
full: true
query:
- - "'mochitest-browser-newtab-trainhop !cf !tsan !ccov !asan !shippable !debug !devedition"
+ - "'mochitest trainhop !cf !tsan !ccov !asan !shippable !debug !devedition !wayland !-32"
devtools:
selector: fuzzy
