commit 77eb67e00d0f67c2afd5f40933ba89a739040775
parent ac744cb20901b5003867123ae70a94e91cf60235
Author: Erik Nordin <enordin@mozilla.com>
Date: Wed, 5 Nov 2025 16:02:30 +0000
Bug 1997612 - Add Translations telemetry rate-limit tests r=translations-reviewers,gregtatum
This patch adds an XPCShell test to ensure that the random number generation
of the Translations telemetry rate limits is behaving uniformly, and that
the rate limits themselves are behaving as expected for each channel.
Differential Revision: https://phabricator.services.mozilla.com/D270929
Diffstat:
3 files changed, 144 insertions(+), 6 deletions(-)
diff --git a/toolkit/components/translations/TranslationsTelemetry.sys.mjs b/toolkit/components/translations/TranslationsTelemetry.sys.mjs
@@ -86,7 +86,7 @@ export class TranslationsTelemetry {
*
* @returns {number}
*/
- static #randomRoll() {
+ static randomRoll() {
// Generate a uniformly-distributed, random u32 in #RANDOM_VALUE.
crypto.getRandomValues(TranslationsTelemetry.#RANDOM_VALUE);
@@ -120,22 +120,34 @@ export class TranslationsTelemetry {
* Channels omitted from the mapping are never skipped.
* @param {FlowContext} [flowContext]
* - The context that contains the flow id and a random roll for the entire flow.
+ * @param {UpdateChannel} [channel]
+ * - The update channel whose sampling policy should be applied. Defaults to the
+ * application-wide update channel, but may be overridden for testing.
*
* @returns {boolean} True if the event should be skipped for this channel, otherwise false.
*/
- static shouldSkipSample(sampleRates, flowContext) {
+ static shouldSkipSample(
+ sampleRates,
+ flowContext,
+ channel = AppConstants.MOZ_UPDATE_CHANNEL
+ ) {
if (Cu.isInAutomation && !sampleRates.applyInAutomation) {
// Do no skip any samples in automation, unless it is explicitly requested.
return false;
}
- const channel = AppConstants.MOZ_UPDATE_CHANNEL;
+ if (channel !== AppConstants.MOZ_UPDATE_CHANNEL && !Cu.isInAutomation) {
+ throw new Error(
+ `Channel "${AppConstants.MOZ_UPDATE_CHANNEL}" was overridden as "${channel}" outside of testing.`
+ );
+ }
+
const sampleRate = sampleRates[channel];
let randomRoll;
if (lazy.console?.shouldLog("Debug")) {
randomRoll =
- flowContext?.randomRoll ?? TranslationsTelemetry.#randomRoll();
+ flowContext?.randomRoll ?? TranslationsTelemetry.randomRoll();
lazy.console.debug({
randomRoll: randomRoll.toFixed(8),
@@ -169,7 +181,7 @@ export class TranslationsTelemetry {
if (randomRoll === undefined) {
randomRoll =
- flowContext?.randomRoll ?? TranslationsTelemetry.#randomRoll();
+ flowContext?.randomRoll ?? TranslationsTelemetry.randomRoll();
}
return randomRoll >= sampleRate;
@@ -228,7 +240,7 @@ export class TranslationsTelemetry {
static createFlowContext() {
const flowContext = {
flowId: crypto.randomUUID(),
- randomRoll: TranslationsTelemetry.#randomRoll(),
+ randomRoll: TranslationsTelemetry.randomRoll(),
};
TranslationsTelemetry.#flowContext = flowContext;
diff --git a/toolkit/components/translations/tests/unit/test_telemetry_sampling.js b/toolkit/components/translations/tests/unit/test_telemetry_sampling.js
@@ -0,0 +1,123 @@
+/* Any copyright is dedicated to the Public Domain.
+https://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+const { TranslationsTelemetry } = ChromeUtils.importESModule(
+ "chrome://global/content/translations/TranslationsTelemetry.sys.mjs"
+);
+
+add_task(function test_sampling_across_channels() {
+ // prettier-ignore
+ const sampleRates = {
+ applyInAutomation: true,
+ default: 1 / 10,
+ nightly: 1 / 100,
+ beta: 1 / 1_000,
+ esr: 1 / 10_000,
+ release: 1 / 100_000,
+ };
+
+ // prettier-ignore
+ const outcomes = {
+ default: { recordedCount: 0, skippedCount: 0 },
+ nightly: { recordedCount: 0, skippedCount: 0 },
+ beta: { recordedCount: 0, skippedCount: 0 },
+ esr: { recordedCount: 0, skippedCount: 0 },
+ release: { recordedCount: 0, skippedCount: 0 },
+ };
+
+ const channels = Object.keys(outcomes);
+ const iterations = 1_000_000;
+
+ info(`Collecting ${iterations} outcomes for each channel.`);
+ for (let iteration = 0; iteration < iterations; iteration++) {
+ const flowContext = TranslationsTelemetry.createFlowContext();
+
+ for (const channel of channels) {
+ const shouldSkip = TranslationsTelemetry.shouldSkipSample(
+ sampleRates,
+ flowContext,
+ channel
+ );
+
+ if (shouldSkip) {
+ outcomes[channel].skippedCount++;
+ } else {
+ outcomes[channel].recordedCount++;
+ }
+ }
+ }
+
+ info(
+ `Checking that all ${iterations} outcomes are present for each channel.`
+ );
+ for (const channel of channels) {
+ const { recordedCount: recourdedCount, skippedCount } = outcomes[channel];
+ equal(
+ recourdedCount + skippedCount,
+ iterations,
+ `The total outcomes for the "${channel}" channel should cover every iteration.`
+ );
+ }
+
+ info(
+ `Checking that channels with a higher probability to record have more recorded events.`
+ );
+ for (let index = 0; index < channels.length - 1; index++) {
+ const current = channels[index];
+ const next = channels[index + 1];
+
+ // Since the are denominators in this test increase so drastically with each subsequent channel,
+ // these assertions are nearly impossible to fail, even with non-deterministically seeded randomness.
+ Assert.greater(
+ outcomes[current].recordedCount,
+ outcomes[next].recordedCount,
+ `The recorded count for the "${current}" channel should be greater than the "${next}" channel.`
+ );
+ }
+
+ // Each channel has its own probability to record an event or skip an event.
+ // For a large number of iterations, the number of recorded events should follow a binomial distribution.
+ //
+ // - https://en.wikipedia.org/wiki/Binomial_distribution
+ //
+ // The variance of a binomial is defined as n * p * (1 - p),
+ // where n is the number of iterations and p is the probability to record an event.
+ //
+ // The square root of that variance is the standard deviation.
+ //
+ // We will use a tolerance of +/- 6x standard deviation to ensure that our randomness does not cause excessive intermittent failures.
+ //
+ // - https://en.wikipedia.org/wiki/68–95–99.7_rule
+ //
+ // Within a normal distribution, a tolerance of +/- 3x standard deviation covers 99.7% of outcomes.
+ // Accordig to the table on the linked Wikipedia page, +/- 6x should fail only 1 out of 506,797,346 times.
+ //
+ // Using non-determinstically seeded randomness in this test will help ensure that our in-production rng behaves as expeced.
+ const multiplier = 6;
+
+ info(
+ "Checking that each channel's recorded event count is within the expected statistical tolerance."
+ );
+ for (const channel of channels) {
+ const sampleRate = sampleRates[channel];
+ const { recordedCount } = outcomes[channel];
+ const expectedRecordedCount = iterations * sampleRate;
+ const deviation = Math.abs(recordedCount - expectedRecordedCount);
+ const standardDeviation = Math.sqrt(
+ iterations * sampleRate * (1 - sampleRate)
+ );
+ const tolerance = multiplier * standardDeviation;
+
+ info(
+ `Channel("${channel}"): expected(${expectedRecordedCount}), recorded(${recordedCount}), deviation(${deviation.toFixed(1)}), tolerance(${tolerance.toFixed(1)})`
+ );
+
+ Assert.lessOrEqual(
+ deviation,
+ tolerance,
+ `The recorded count for "${channel}" remains within +/- ${multiplier}x of the expected standard deviation.`
+ );
+ }
+});
diff --git a/toolkit/components/translations/tests/unit/xpcshell.toml b/toolkit/components/translations/tests/unit/xpcshell.toml
@@ -3,3 +3,6 @@ head = ""
firefox-appdir = "browser"
["test_cld2.js"]
+
+["test_telemetry_sampling.js"]
+requesttimeoutfactor = 4
