commit 3bdfb972430fec655973514156fab08e428bcc5a
parent 9ef530f45420a3cca49cf29b0a081953c1a883a1
Author: Erik Nordin <enordin@mozilla.com>
Date: Fri, 3 Oct 2025 19:26:50 +0000
Bug 1991761 - Test all architectures in Translations perftests r=perftest-reviewers,mozperftest-reviewers,translations-reviewers,gregtatum,sparky
This patch updates all of the Translations perftest infrastructure
to test each Translations model architecture kind: tiny, base-memory,
and base.
Differential Revision: https://phabricator.services.mozilla.com/D266895
Diffstat:
13 files changed, 665 insertions(+), 58 deletions(-)
diff --git a/browser/components/translations/tests/browser/browser_translations_perf_base.js b/browser/components/translations/tests/browser/browser_translations_perf_base.js
@@ -0,0 +1,110 @@
+/* Any copyright is dedicated to the Public Domain.
+ https://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * This metadata schema is parsed by the perftest infrastructure.
+ *
+ * The perftest runner then scrapes the logs for a JSON results matching this schema,
+ * which are logged by the TranslationsBencher class.
+ *
+ * @see {TranslationsBencher.Journal}
+ */
+const perfMetadata = {
+ owner: "Translations Team",
+ name: "Full-Page Translations Base Model",
+ description:
+ "Tests the performance of Full Page Translations with a base-architecture model",
+ options: {
+ default: {
+ perfherder: true,
+ perfherder_metrics: [
+ {
+ name: "engine-init-time",
+ unit: "ms",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "words-per-second",
+ unit: "WPS",
+ shouldAlert: true,
+ lowerIsBetter: false,
+ },
+ {
+ name: "tokens-per-second",
+ unit: "TPS",
+ shouldAlert: true,
+ lowerIsBetter: false,
+ },
+ {
+ name: "peak-parent-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "stabilized-parent-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "post-gc-parent-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "peak-inference-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "stabilized-inference-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "post-gc-inference-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "total-translation-time",
+ unit: "s",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ ],
+ verbose: true,
+ manifest: "perftest.toml",
+ manifest_flavor: "browser-chrome",
+ try_platform: ["linux", "mac", "win"],
+ },
+ },
+};
+
+/**
+ * Request a longer timeout for this test.
+ */
+requestLongerTimeout(8);
+
+/**
+ * Runs the translations benchmark tests with a base-architecture model.
+ */
+add_task(async function test_translations_performance_base() {
+ await TranslationsBencher.benchmarkTranslation({
+ page: ENGLISH_BENCHMARK_PAGE_URL,
+ sourceLanguage: "en",
+ targetLanguage: "ru",
+ architecture: "base",
+ speedBenchCount: 5,
+ memoryBenchCount: 5,
+ memorySampleInterval: 50,
+ });
+});
diff --git a/browser/components/translations/tests/browser/browser_translations_perf_basememory.js b/browser/components/translations/tests/browser/browser_translations_perf_basememory.js
@@ -0,0 +1,110 @@
+/* Any copyright is dedicated to the Public Domain.
+ https://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * This metadata schema is parsed by the perftest infrastructure.
+ *
+ * The perftest runner then scrapes the logs for a JSON results matching this schema,
+ * which are logged by the TranslationsBencher class.
+ *
+ * @see {TranslationsBencher.Journal}
+ */
+const perfMetadata = {
+ owner: "Translations Team",
+ name: "Full-Page Translations BaseMemory Model",
+ description:
+ "Tests the performance of Full Page Translations with a base-memory-architecture model",
+ options: {
+ default: {
+ perfherder: true,
+ perfherder_metrics: [
+ {
+ name: "engine-init-time",
+ unit: "ms",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "words-per-second",
+ unit: "WPS",
+ shouldAlert: true,
+ lowerIsBetter: false,
+ },
+ {
+ name: "tokens-per-second",
+ unit: "TPS",
+ shouldAlert: true,
+ lowerIsBetter: false,
+ },
+ {
+ name: "peak-parent-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "stabilized-parent-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "post-gc-parent-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "peak-inference-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "stabilized-inference-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "post-gc-inference-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "total-translation-time",
+ unit: "s",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ ],
+ verbose: true,
+ manifest: "perftest.toml",
+ manifest_flavor: "browser-chrome",
+ try_platform: ["linux", "mac", "win"],
+ },
+ },
+};
+
+/**
+ * Request a longer timeout for this test.
+ */
+requestLongerTimeout(7);
+
+/**
+ * Runs the translations benchmark tests with a base-memory-architecture model.
+ */
+add_task(async function test_translations_performance_basememory() {
+ await TranslationsBencher.benchmarkTranslation({
+ page: ENGLISH_BENCHMARK_PAGE_URL,
+ sourceLanguage: "en",
+ targetLanguage: "ru",
+ architecture: "base-memory",
+ speedBenchCount: 5,
+ memoryBenchCount: 5,
+ memorySampleInterval: 50,
+ });
+});
diff --git a/browser/components/translations/tests/browser/browser_translations_perf_tiny.js b/browser/components/translations/tests/browser/browser_translations_perf_tiny.js
@@ -0,0 +1,110 @@
+/* Any copyright is dedicated to the Public Domain.
+ https://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * This metadata schema is parsed by the perftest infrastructure.
+ *
+ * The perftest runner then scrapes the logs for a JSON results matching this schema,
+ * which are logged by the TranslationsBencher class.
+ *
+ * @see {TranslationsBencher.Journal}
+ */
+const perfMetadata = {
+ owner: "Translations Team",
+ name: "Full-Page Translations Tiny Model",
+ description:
+ "Tests the performance of Full Page Translations with a tiny-architecture model",
+ options: {
+ default: {
+ perfherder: true,
+ perfherder_metrics: [
+ {
+ name: "engine-init-time",
+ unit: "ms",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "words-per-second",
+ unit: "WPS",
+ shouldAlert: true,
+ lowerIsBetter: false,
+ },
+ {
+ name: "tokens-per-second",
+ unit: "TPS",
+ shouldAlert: true,
+ lowerIsBetter: false,
+ },
+ {
+ name: "peak-parent-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "stabilized-parent-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "post-gc-parent-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "peak-inference-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "stabilized-inference-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "post-gc-inference-process-memory-usage",
+ unit: "MiB",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ {
+ name: "total-translation-time",
+ unit: "s",
+ shouldAlert: true,
+ lowerIsBetter: true,
+ },
+ ],
+ verbose: true,
+ manifest: "perftest.toml",
+ manifest_flavor: "browser-chrome",
+ try_platform: ["linux", "mac", "win"],
+ },
+ },
+};
+
+/**
+ * Request a longer timeout for this test.
+ */
+requestLongerTimeout(6);
+
+/**
+ * Runs the translations benchmark tests with a tiny-architecture model.
+ */
+add_task(async function test_translations_performance_tiny() {
+ await TranslationsBencher.benchmarkTranslation({
+ page: ENGLISH_BENCHMARK_PAGE_URL,
+ sourceLanguage: "en",
+ targetLanguage: "ru",
+ architecture: "tiny",
+ speedBenchCount: 5,
+ memoryBenchCount: 5,
+ memorySampleInterval: 50,
+ });
+});
diff --git a/browser/components/translations/tests/browser/perftest.toml b/browser/components/translations/tests/browser/perftest.toml
@@ -5,3 +5,11 @@ support-files = [
"!/toolkit/components/translations/tests/browser/translations-test.mjs",
]
+["browser_translations_perf_base.js"]
+disabled = "Disabled since we want to run this only as perftest, not in regular CI."
+
+["browser_translations_perf_basememory.js"]
+disabled = "Disabled since we want to run this only as perftest, not in regular CI."
+
+["browser_translations_perf_tiny.js"]
+disabled = "Disabled since we want to run this only as perftest, not in regular CI."
diff --git a/python/mozperftest/perfdocs/config.yml b/python/mozperftest/perfdocs/config.yml
@@ -43,7 +43,9 @@ suites:
browser/components/translations/tests/browser:
description: "Performance tests for Translations models on Firefox Desktop"
tests:
- "Full-Page Translation (Spanish to English)": ""
+ "Full-Page Translations Base Model": ""
+ "Full-Page Translations BaseMemory Model": ""
+ "Full-Page Translations Tiny Model": ""
dom/serviceworkers/test/performance:
description: "Performance tests running through Mochitest for Service Workers"
diff --git a/taskcluster/kinds/fetch/translations-fetch.yml b/taskcluster/kinds/fetch/translations-fetch.yml
@@ -2,38 +2,119 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
-translations.inference:
+tr8ns.inference:
description: The bergamot-translator WASM artifact (version 4.0)
fetch:
artifact-name: bergamot-translator.zst
type: static-url
- url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/wasm/bergamot-translator.wasm.zst
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/wasm/bergamot-translator.wasm.zst
sha256: 327fcfc7b7e9d95f6fa0844ebf899cfc05469872ef02d3aef5783182839a6255
size: 1211955
-translations.esen.lex:
- description: The Spanish to English translation lex artifact (version 3.0)
+tr8ns.T.esen.lex:
+ description: Spanish to English lex artifact (tiny)
fetch:
- artifact-name: lex.50.50.esen.s2t.bin.zst
+ artifact-name: tiny.lex.50.50.esen.s2t.bin.zst
type: static-url
- url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/models/es-en/test_student_preemption_None/exported/lex.50.50.esen.s2t.bin.zst
- sha256: 165bb333501d93cb7fe04b958c87f07863f301f4a0e41634906b1b4ad9c03214
- size: 1531056
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/tiny/es-en/lex.50.50.esen.s2t.bin.zst
+ sha256: 04f64a043ed724dc93aea1e5a322a26f0b1be937849a066d4141225634901aa5
+ size: 1554839
-translations.esen.model:
- description: The Spanish to English translation model artifact (version 3.0)
+tr8ns.T.esen.model:
+ description: Spanish to English model artifact (tiny)
fetch:
- artifact-name: model.esen.intgemm.alphas.bin.zst
+ artifact-name: tiny.model.esen.intgemm.alphas.bin.zst
type: static-url
- url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/models/es-en/test_student_preemption_None/exported/model.esen.intgemm.alphas.bin.zst
- sha256: fbd3e63219604b93c4d7d98bf045f6239b4a8175d540a9e6b2964acd25c469a7
- size: 13112493
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/tiny/es-en/model.esen.intgemm.alphas.bin.zst
+ sha256: 4b8a8fbd065a74af3173aa6df8480a5977f2b77e0a7ea1b978174e98a45e58f1
+ size: 13112217
-translations.esen.vocab:
- description: The Spanish to English translation vocab artifact (version 3.0)
+tr8ns.T.esen.vocab:
+ description: Spanish to English vocab artifact (tiny)
fetch:
- artifact-name: vocab.esen.spm.zst
+ artifact-name: tiny.vocab.esen.spm.zst
type: static-url
- url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/models/es-en/test_student_preemption_None/exported/vocab.esen.spm.zst
- sha256: aac1f48284c4ca74b7f21a9363049315eeb9074c1e6ed4bf8f58e6600315e35f
- size: 351725
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/tiny/es-en/vocab.esen.spm.zst
+ sha256: 872d8e312297dfcbcc20d3d94b42d0cb872dcabfa16195d103bfc96c441d70e4
+ size: 351388
+
+tr8ns.T.enru.lex:
+ description: English to Russian lex artifact (tiny)
+ fetch:
+ artifact-name: tiny.lex.50.50.enru.s2t.bin.zst
+ type: static-url
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/tiny/en-ru/lex.50.50.enru.s2t.bin.zst
+ sha256: d5a33e067e7a7b2a1168f70d403424c4e68c539c333837bde05b150f83c39e8b
+ size: 990112
+
+tr8ns.T.enru.model:
+ description: English to Russian model artifact (tiny)
+ fetch:
+ artifact-name: tiny.model.enru.intgemm.alphas.bin.zst
+ type: static-url
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/tiny/en-ru/model.enru.intgemm.alphas.bin.zst
+ sha256: 416e1ecb3077a01cc899f3f8d86cee1a5dfc55c9621c5e1de454aa59ad0c7d1b
+ size: 12107280
+
+tr8ns.T.enru.vocab:
+ description: English to Russian vocab artifact (tiny)
+ fetch:
+ artifact-name: tiny.vocab.enru.spm.zst
+ type: static-url
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/tiny/en-ru/vocab.enru.spm.zst
+ sha256: 088a38d8b5faf6315b284e3088714f70a8c79438496b15826b6c62496503c447
+ size: 345737
+
+tr8ns.BM.enru.lex:
+ description: English to Russian lex artifact (base memory)
+ fetch:
+ artifact-name: base-memory.lex.50.50.enru.s2t.bin.zst
+ type: static-url
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/base-memory/en-ru/lex.50.50.enru.s2t.bin.zst
+ sha256: d7998ec23c850255c2436bab1a248944877f63bf1842e02f67e50807d172b1bf
+ size: 1013057
+
+tr8ns.BM.enru.model:
+ description: English to Russian model artifact (base memory)
+ fetch:
+ artifact-name: base-memory.model.enru.intgemm.alphas.bin.zst
+ type: static-url
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/base-memory/en-ru/model.enru.intgemm.alphas.bin.zst
+ sha256: a55b2af86653d6e5f4501c674f3806535734f93540e83db19bffdafe456d0887
+ size: 21044601
+
+tr8ns.BM.enru.vocab:
+ description: English to Russian vocab artifact (base memory)
+ fetch:
+ artifact-name: base-memory.vocab.enru.spm.zst
+ type: static-url
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/base-memory/en-ru/vocab.enru.spm.zst
+ sha256: 088a38d8b5faf6315b284e3088714f70a8c79438496b15826b6c62496503c447
+ size: 345737
+
+tr8ns.B.enru.lex:
+ description: English to Russian lex artifact (base)
+ fetch:
+ artifact-name: base.lex.50.50.enru.s2t.bin.zst
+ type: static-url
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/base/en-ru/lex.50.50.enru.s2t.bin.zst
+ sha256: d5a33e067e7a7b2a1168f70d403424c4e68c539c333837bde05b150f83c39e8b
+ size: 990112
+
+tr8ns.B.enru.model:
+ description: English to Russian model artifact (base)
+ fetch:
+ artifact-name: base.model.enru.intgemm.alphas.bin.zst
+ type: static-url
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/base/en-ru/model.enru.intgemm.alphas.bin.zst
+ sha256: 8a9ee496752361b7d7dd449dab046929880d21ab3ebcd2a261f076441d52dc30
+ size: 29744901
+
+tr8ns.B.enru.vocab:
+ description: English to Russian vocab artifact (base)
+ fetch:
+ artifact-name: base.vocab.enru.spm.zst
+ type: static-url
+ url: https://storage.googleapis.com/moz-fx-translations-data--303e-prod-translations-data/firefox-ci/models/base/en-ru/vocab.enru.spm.zst
+ sha256: 088a38d8b5faf6315b284e3088714f70a8c79438496b15826b6c62496503c447
+ size: 345737
diff --git a/taskcluster/kinds/mochitest/kind.yml b/taskcluster/kinds/mochitest/kind.yml
@@ -552,10 +552,10 @@ tasks:
virtualization: virtual
fetches:
fetch:
- - translations.inference
- - translations.esen.lex
- - translations.esen.model
- - translations.esen.vocab
+ - tr8ns.inference
+ - tr8ns.T.esen.lex
+ - tr8ns.T.esen.model
+ - tr8ns.T.esen.vocab
mochitest-chrome:
description: "Mochitest chrome run"
diff --git a/taskcluster/kinds/perftest/linux.yml b/taskcluster/kinds/perftest/linux.yml
@@ -570,16 +570,16 @@ ml-perf-autofill:
--hooks toolkit/components/ml/tests/tools/hooks_local_hub.py
toolkit/components/ml/tests/browser/browser_ml_autofill_perf.js
-tr8ns-perf-es-en:
+tr8ns-perf-tiny:
fetches:
fetch:
- - translations.inference
- - translations.esen.lex
- - translations.esen.model
- - translations.esen.vocab
- description: Run Full-Page Translations with the Spanish-to-English model
+ - tr8ns.inference
+ - tr8ns.T.enru.lex
+ - tr8ns.T.enru.model
+ - tr8ns.T.enru.vocab
+ description: Run Full-Page Translations with a tiny-architecture model
treeherder:
- symbol: perftest(tr8ns-perf-es-en)
+ symbol: perftest(tr8ns-perf-T)
tier: 2
attributes:
batch: false
@@ -593,7 +593,57 @@ tr8ns-perf-es-en:
--mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin
--flavor mochitest
--output $MOZ_FETCHES_DIR/../artifacts
- browser/components/translations/tests/browser/browser_translations_perf_es_en.js
+ browser/components/translations/tests/browser/browser_translations_perf_tiny.js
+
+tr8ns-perf-basememory:
+ fetches:
+ fetch:
+ - tr8ns.inference
+ - tr8ns.BM.enru.lex
+ - tr8ns.BM.enru.model
+ - tr8ns.BM.enru.vocab
+ description: Run Full-Page Translations with a base-memory-architecture model
+ treeherder:
+ symbol: perftest(tr8ns-perf-BM)
+ tier: 2
+ attributes:
+ batch: false
+ cron: false
+ run-on-projects: [autoland, mozilla-central]
+ run:
+ command: >-
+ mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+ cd $MOZ_FETCHES_DIR &&
+ python3 python/mozperftest/mozperftest/runner.py
+ --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin
+ --flavor mochitest
+ --output $MOZ_FETCHES_DIR/../artifacts
+ browser/components/translations/tests/browser/browser_translations_perf_basememory.js
+
+tr8ns-perf-base:
+ fetches:
+ fetch:
+ - tr8ns.inference
+ - tr8ns.B.enru.lex
+ - tr8ns.B.enru.model
+ - tr8ns.B.enru.vocab
+ description: Run Full-Page Translations with a base-architecture model
+ treeherder:
+ symbol: perftest(tr8ns-perf-B)
+ tier: 2
+ attributes:
+ batch: false
+ cron: false
+ run-on-projects: [autoland, mozilla-central]
+ run:
+ command: >-
+ mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+ cd $MOZ_FETCHES_DIR &&
+ python3 python/mozperftest/mozperftest/runner.py
+ --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox-bin
+ --flavor mochitest
+ --output $MOZ_FETCHES_DIR/../artifacts
+ browser/components/translations/tests/browser/browser_translations_perf_base.js
ml-perf-smart-tab:
fetches:
diff --git a/taskcluster/kinds/perftest/macosx.yml b/taskcluster/kinds/perftest/macosx.yml
@@ -520,16 +520,16 @@ ml-perf-autofill:
--hooks toolkit/components/ml/tests/tools/hooks_local_hub.py
toolkit/components/ml/tests/browser/browser_ml_autofill_perf.js
-tr8ns-perf-es-en:
+tr8ns-perf-tiny:
fetches:
fetch:
- - translations.inference
- - translations.esen.lex
- - translations.esen.model
- - translations.esen.vocab
- description: Run Full-Page Translations with the Spanish-to-English model
+ - tr8ns.inference
+ - tr8ns.T.enru.lex
+ - tr8ns.T.enru.model
+ - tr8ns.T.enru.vocab
+ description: Run Full-Page Translations with a tiny-architecture model
treeherder:
- symbol: perftest(tr8ns-perf-es-en)
+ symbol: perftest(tr8ns-perf-T)
tier: 2
attributes:
batch: false
@@ -543,7 +543,57 @@ tr8ns-perf-es-en:
--mochitest-binary ${MOZ_FETCHES_DIR}/target.dmg
--flavor mochitest
--output $MOZ_FETCHES_DIR/../artifacts
- browser/components/translations/tests/browser/browser_translations_perf_es_en.js
+ browser/components/translations/tests/browser/browser_translations_perf_tiny.js
+
+tr8ns-perf-basememory:
+ fetches:
+ fetch:
+ - tr8ns.inference
+ - tr8ns.BM.enru.lex
+ - tr8ns.BM.enru.model
+ - tr8ns.BM.enru.vocab
+ description: Run Full-Page Translations with a base-memory-architecture model
+ treeherder:
+ symbol: perftest(tr8ns-perf-BM)
+ tier: 2
+ attributes:
+ batch: false
+ cron: false
+ run-on-projects: [autoland, mozilla-central]
+ run:
+ command: >-
+ mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+ cd $MOZ_FETCHES_DIR &&
+ python3 python/mozperftest/mozperftest/runner.py
+ --mochitest-binary ${MOZ_FETCHES_DIR}/target.dmg
+ --flavor mochitest
+ --output $MOZ_FETCHES_DIR/../artifacts
+ browser/components/translations/tests/browser/browser_translations_perf_basememory.js
+
+tr8ns-perf-base:
+ fetches:
+ fetch:
+ - tr8ns.inference
+ - tr8ns.B.enru.lex
+ - tr8ns.B.enru.model
+ - tr8ns.B.enru.vocab
+ description: Run Full-Page Translations with a base-architecture model
+ treeherder:
+ symbol: perftest(tr8ns-perf-B)
+ tier: 2
+ attributes:
+ batch: false
+ cron: false
+ run-on-projects: [autoland, mozilla-central]
+ run:
+ command: >-
+ mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+ cd $MOZ_FETCHES_DIR &&
+ python3 python/mozperftest/mozperftest/runner.py
+ --mochitest-binary ${MOZ_FETCHES_DIR}/target.dmg
+ --flavor mochitest
+ --output $MOZ_FETCHES_DIR/../artifacts
+ browser/components/translations/tests/browser/browser_translations_perf_base.js
ml-perf-smart-tab:
fetches:
diff --git a/taskcluster/kinds/perftest/windows11-24h2.yml b/taskcluster/kinds/perftest/windows11-24h2.yml
@@ -468,16 +468,16 @@ ml-perf-autofill:
--hooks toolkit/components/ml/tests/tools/hooks_local_hub.py
toolkit/components/ml/tests/browser/browser_ml_autofill_perf.js
-tr8ns-perf-es-en:
+tr8ns-perf-tiny:
fetches:
fetch:
- - translations.inference
- - translations.esen.lex
- - translations.esen.model
- - translations.esen.vocab
- description: Run Full-Page Translations with the Spanish-to-English model
+ - tr8ns.inference
+ - tr8ns.T.enru.lex
+ - tr8ns.T.enru.model
+ - tr8ns.T.enru.vocab
+ description: Run Full-Page Translations with a tiny-architecture model
treeherder:
- symbol: perftest(tr8ns-perf-es-en)
+ symbol: perftest(tr8ns-perf-T)
tier: 2
attributes:
batch: false
@@ -491,7 +491,57 @@ tr8ns-perf-es-en:
--mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe
--flavor mochitest
--output $MOZ_FETCHES_DIR/../artifacts
- browser/components/translations/tests/browser/browser_translations_perf_es_en.js
+ browser/components/translations/tests/browser/browser_translations_perf_tiny.js
+
+tr8ns-perf-basememory:
+ fetches:
+ fetch:
+ - tr8ns.inference
+ - tr8ns.BM.enru.lex
+ - tr8ns.BM.enru.model
+ - tr8ns.BM.enru.vocab
+ description: Run Full-Page Translations with a base-memory-architecture model
+ treeherder:
+ symbol: perftest(tr8ns-perf-BM)
+ tier: 2
+ attributes:
+ batch: false
+ cron: false
+ run-on-projects: [autoland, mozilla-central]
+ run:
+ command: >-
+ mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+ cd $MOZ_FETCHES_DIR &&
+ python3 python/mozperftest/mozperftest/runner.py
+ --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe
+ --flavor mochitest
+ --output $MOZ_FETCHES_DIR/../artifacts
+ browser/components/translations/tests/browser/browser_translations_perf_basememory.js
+
+tr8ns-perf-base:
+ fetches:
+ fetch:
+ - tr8ns.inference
+ - tr8ns.B.enru.lex
+ - tr8ns.B.enru.model
+ - tr8ns.B.enru.vocab
+ description: Run Full-Page Translations with a base-architecture model
+ treeherder:
+ symbol: perftest(tr8ns-perf-B)
+ tier: 2
+ attributes:
+ batch: false
+ cron: false
+ run-on-projects: [autoland, mozilla-central]
+ run:
+ command: >-
+ mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
+ cd $MOZ_FETCHES_DIR &&
+ python3 python/mozperftest/mozperftest/runner.py
+ --mochitest-binary ${MOZ_FETCHES_DIR}/firefox/firefox.exe
+ --flavor mochitest
+ --output $MOZ_FETCHES_DIR/../artifacts
+ browser/components/translations/tests/browser/browser_translations_perf_base.js
ml-perf-smart-tab:
fetches:
diff --git a/taskcluster/kinds/test/misc.yml b/taskcluster/kinds/test/misc.yml
@@ -152,10 +152,10 @@ test-verify:
- linux64-hostutils
fetch:
- ort.jsep.wasm
- - translations.inference
- - translations.esen.lex
- - translations.esen.model
- - translations.esen.vocab
+ - tr8ns.inference
+ - tr8ns.T.esen.lex
+ - tr8ns.T.esen.model
+ - tr8ns.T.esen.vocab
test-verify-gpu:
description: "Extra verification of tests modified on this push on gpu instances"
diff --git a/testing/perfdocs/generated/mozperftest.rst b/testing/perfdocs/generated/mozperftest.rst
@@ -114,11 +114,47 @@ browser/components/translations/tests/browser
---------------------------------------------
Performance tests for Translations models on Firefox Desktop
-browser_translations_perf_es_en.js
-==================================
+browser_translations_perf_base.js
+=================================
+
+:owner: Translations Team
+:name: Full-Page Translations Base Model
+:Default options:
+
+::
+
+ --perfherder
+ --perfherder-metrics name:engine-init-time,unit:ms,shouldAlert:True,lowerIsBetter:True, name:words-per-second,unit:WPS,shouldAlert:True,lowerIsBetter:False, name:tokens-per-second,unit:TPS,shouldAlert:True,lowerIsBetter:False, name:peak-parent-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:stabilized-parent-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:post-gc-parent-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:peak-inference-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:stabilized-inference-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:post-gc-inference-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:total-translation-time,unit:s,shouldAlert:True,lowerIsBetter:True
+ --verbose
+ --manifest perftest.toml
+ --manifest-flavor browser-chrome
+ --try-platform linux, mac, win
+
+**Tests the performance of Full Page Translations with a base-architecture model**
+
+browser_translations_perf_basememory.js
+=======================================
+
+:owner: Translations Team
+:name: Full-Page Translations BaseMemory Model
+:Default options:
+
+::
+
+ --perfherder
+ --perfherder-metrics name:engine-init-time,unit:ms,shouldAlert:True,lowerIsBetter:True, name:words-per-second,unit:WPS,shouldAlert:True,lowerIsBetter:False, name:tokens-per-second,unit:TPS,shouldAlert:True,lowerIsBetter:False, name:peak-parent-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:stabilized-parent-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:post-gc-parent-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:peak-inference-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:stabilized-inference-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:post-gc-inference-process-memory-usage,unit:MiB,shouldAlert:True,lowerIsBetter:True, name:total-translation-time,unit:s,shouldAlert:True,lowerIsBetter:True
+ --verbose
+ --manifest perftest.toml
+ --manifest-flavor browser-chrome
+ --try-platform linux, mac, win
+
+**Tests the performance of Full Page Translations with a base-memory-architecture model**
+
+browser_translations_perf_tiny.js
+=================================
:owner: Translations Team
-:name: Full-Page Translation (Spanish to English)
+:name: Full-Page Translations Tiny Model
:Default options:
::
@@ -130,7 +166,7 @@ browser_translations_perf_es_en.js
--manifest-flavor browser-chrome
--try-platform linux, mac, win
-**Tests the speed of Full Page Translations using the Spanish-to-English model.**
+**Tests the performance of Full Page Translations with a tiny-architecture model**
dom/serviceworkers/test/performance
diff --git a/toolkit/components/translations/docs/resources/03_bergamot.md b/toolkit/components/translations/docs/resources/03_bergamot.md
@@ -68,7 +68,7 @@ If the Bergamot Translator has a breaking change, then the `BERGAMOT_MAJOR_VERSI
- Find the record in Remote Settings' [translations-wasm](https://remote-settings.mozilla.org/v1/admin/#/buckets/main-workspace/collections/translations-wasm/records) view.
- At this point the record could still be in Preview, as the attachment won't change once it's approved.
- Double click the newest record.
- - Update the `url`, `sha256`, `size`, and `description` fields for `translations.inference.fetch` in `translations-fetch.yml`.
+ - Update the `url`, `sha256`, `size`, and `description` fields for `tr8ns.inference` in `translations-fetch.yml`.
- The `size` can be found via `curl -sL --head $url` and the `content-length`.
1. Publish to Nightly
