commit 590be5def684a944bf3d30a97d74e0f2bf5dcb2b
parent e3c98218ce011bca92c043e3efd4d9661c1b16a5
Author: cypherpunks1 <2478-cypherpunks1@gitlab.torproject.org>
Date: Wed, 28 Aug 2024 05:36:48 -0800
BB 42730: Patch RemoteSettings to use only local dumps as a data source
Diffstat:
7 files changed, 99 insertions(+), 40 deletions(-)
diff --git a/browser/components/BrowserGlue.sys.mjs b/browser/components/BrowserGlue.sys.mjs
@@ -1350,6 +1350,11 @@ BrowserGlue.prototype = {
this._addBreachesSyncHandler();
}.bind(this),
+ function RemoteSettingsPollChanges() {
+ // Support clients that use the "sync" event or "remote-settings:changes-poll-end".
+ lazy.RemoteSettings.pollChanges({ trigger: "timer" });
+ },
+
function searchBackgroundChecks() {
Services.search.runBackgroundChecks();
},
diff --git a/mobile/shared/chrome/geckoview/geckoview.js b/mobile/shared/chrome/geckoview/geckoview.js
@@ -21,6 +21,7 @@ ChromeUtils.defineESModuleGetters(this, {
InitializationTracker: "resource://gre/modules/GeckoViewTelemetry.sys.mjs",
RemoteSecuritySettings:
"resource://gre/modules/psm/RemoteSecuritySettings.sys.mjs",
+ RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
SafeBrowsing: "resource://gre/modules/SafeBrowsing.sys.mjs",
CaptchaDetectionPingUtils:
"resource://gre/modules/CaptchaDetectionPingUtils.sys.mjs",
@@ -930,6 +931,10 @@ function startup() {
CaptchaDetectionPingUtils.init();
});
+ InitLater(() => {
+ RemoteSettings.pollChanges({ trigger: "timer" });
+ });
+
// This should always go last, since the idle tasks (except for the ones with
// timeouts) should execute in order. Note that this observer notification is
// not guaranteed to fire, since the window could close before we get here.
diff --git a/services/settings/Attachments.sys.mjs b/services/settings/Attachments.sys.mjs
@@ -359,6 +359,10 @@ export class Downloader {
fallbackToDump = false;
}
+ avoidDownload = true;
+ fallbackToCache = true;
+ fallbackToDump = true;
+
const dumpInfo = new LazyRecordAndBuffer(() =>
this._readAttachmentDump(attachmentId)
);
@@ -521,6 +525,8 @@ export class Downloader {
attachment: { location, hash, size },
} = record;
+ return (await this.#fetchAttachment(record)).buffer;
+ // eslint-disable-next-line no-unreachable
let baseURL;
try {
baseURL = await lazy.Utils.baseAttachmentsURL();
diff --git a/services/settings/RemoteSettingsClient.sys.mjs b/services/settings/RemoteSettingsClient.sys.mjs
@@ -445,11 +445,19 @@ export class RemoteSettingsClient extends EventEmitter {
order = "", // not sorted by default.
dumpFallback = true,
emptyListFallback = true,
- forceSync = false,
loadDumpIfNewer = true,
- syncIfEmpty = true,
} = options;
- let { verifySignature = false } = options;
+
+ const hasLocalDump = await lazy.Utils.hasLocalDump(
+ this.bucketName,
+ this.collectionName
+ );
+ if (!hasLocalDump) {
+ return [];
+ }
+ const forceSync = false;
+ const syncIfEmpty = true;
+ let verifySignature = false;
const hasParallelCall = !!this._importingPromise;
let data;
@@ -648,6 +656,10 @@ export class RemoteSettingsClient extends EventEmitter {
* @param {object} options See #maybeSync() options.
*/
async sync(options) {
+ if (AppConstants.BASE_BROWSER_VERSION) {
+ return;
+ }
+
if (lazy.Utils.shouldSkipRemoteActivityDueToTests) {
lazy.console.debug(`${this.identifier} Skip sync() due to tests.`);
return;
@@ -731,7 +743,7 @@ export class RemoteSettingsClient extends EventEmitter {
let thrownError = null;
try {
// If network is offline, we can't synchronize.
- if (lazy.Utils.isOffline) {
+ if (!AppConstants.BASE_BROWSER_VERSION && lazy.Utils.isOffline) {
throw new RemoteSettingsClient.NetworkOfflineError();
}
@@ -1152,8 +1164,6 @@ export class RemoteSettingsClient extends EventEmitter {
) {
const hasLocalData = localTimestamp !== null;
const { retry = false } = options;
- // On retry, we fully re-fetch the collection (no `?_since`).
- const since = retry || !hasLocalData ? undefined : `"${localTimestamp}"`;
// Define an executor that will verify the signature of the local data.
const verifySignatureLocalData = (resolve, reject) => {
@@ -1181,45 +1191,26 @@ export class RemoteSettingsClient extends EventEmitter {
});
};
- // Fetch collection metadata and list of changes from server.
- lazy.console.debug(
- `${this.identifier} Fetch changes from server (expected=${expectedTimestamp}, since=${since})`
- );
- const { metadata, remoteTimestamp, remoteRecords } =
- await this._fetchChangeset(expectedTimestamp, since);
-
- lazy.console.debug(
- `${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp} (expected: ${expectedTimestamp})`
- );
+ let metadata, remoteTimestamp;
- if (remoteTimestamp < localTimestamp) {
- // This should never happen. Unless the CDN serves stale data.
- // If the local data is valid, then we can safely ignore this stage remote changeset.
- const localTrustworthy = await new Promise(verifySignatureLocalData);
- if (localTrustworthy) {
- lazy.console.info(`${this.identifier} CDN served staled data, ignore.`);
- return {
- current: localRecords,
- created: [],
- updated: [],
- deleted: [],
- };
- }
- // Otherwise, continue with importing, since we prefer stale data over corrupt/tempered.
- lazy.console.warn(
- `${this.identifier} CDN served staled data, but local data is corrupted, import anyway.`
- );
+ try {
+ await this._importJSONDump();
+ } catch (e) {
+ return {
+ current: localRecords,
+ created: [],
+ updated: [],
+ deleted: [],
+ };
}
- await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, {
- clear: retry,
- });
-
// Read the new local data, after updating.
const newLocal = await this.db.list();
const newRecords = newLocal.map(r => this._cleanLocalFields(r));
// And verify the signature on what is now stored.
- if (this.verifySignature) {
+ if (metadata === undefined) {
+ // When working only with dumps, we do not have signatures.
+ } else if (this.verifySignature) {
try {
await this.validateCollectionSignature(
newRecords,
diff --git a/services/settings/dumps/gen_last_modified.py b/services/settings/dumps/gen_last_modified.py
@@ -63,8 +63,10 @@ def main(output):
dumps_locations = []
if buildconfig.substs["MOZ_BUILD_APP"] == "browser":
dumps_locations += ["services/settings/dumps/"]
+ dumps_locations += ["services/settings/static-dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/android":
dumps_locations += ["services/settings/dumps/"]
+ dumps_locations += ["services/settings/static-dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/ios":
dumps_locations += ["services/settings/dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "comm/mail":
diff --git a/services/settings/remote-settings.sys.mjs b/services/settings/remote-settings.sys.mjs
@@ -150,6 +150,7 @@ export async function jexlFilterCreator(environment, collectionName) {
function remoteSettingsFunction() {
const _clients = new Map();
let _invalidatePolling = false;
+ let _initialized = false;
// If not explicitly specified, use the default signer.
const defaultOptions = {
@@ -219,6 +220,43 @@ function remoteSettingsFunction() {
}
/**
+ * Internal helper that checks all registered remote settings clients for the presence
+ * of a newer local data dump. If a local dump is found and its last modified timestamp
+ * is more recent than the client's current data, imports the dump by triggering a sync.
+ * Notifies observers if any data was imported from a dump.
+ *
+ * @param {string} [trigger="manual"] - The reason or source for triggering the import (e.g., "manual", "startup").
+ * @returns {Promise<void>} Resolves when the import process is complete.
+ * @private
+ */
+ async function _maybeImportFromLocalDump(trigger = "manual") {
+ let importedFromDump = false;
+ for (const client of _clients.values()) {
+ const hasLocalDump = await lazy.Utils.hasLocalDump(
+ client.bucketName,
+ client.collectionName
+ );
+ if (hasLocalDump) {
+ const lastModified = await client.getLastModified();
+ const lastModifiedDump = await lazy.Utils.getLocalDumpLastModified(
+ client.bucketName,
+ client.collectionName
+ );
+ if (lastModified < lastModifiedDump) {
+ await client.maybeSync(lastModifiedDump, {
+ loadDump: true,
+ trigger,
+ });
+ importedFromDump = true;
+ }
+ }
+ }
+ if (importedFromDump) {
+ Services.obs.notifyObservers(null, "remote-settings:changes-poll-end");
+ }
+ }
+
+ /**
* Helper to introspect the synchronization history and determine whether it is
* consistently failing and thus, broken.
*
@@ -352,6 +390,16 @@ function remoteSettingsFunction() {
trigger = "manual",
full = false,
} = {}) => {
+ if (AppConstants.BASE_BROWSER_VERSION) {
+ // Called multiple times on GeckoView due to bug 1730026
+ if (_initialized) {
+ return;
+ }
+ _initialized = true;
+ _maybeImportFromLocalDump(trigger);
+ return;
+ }
+
if (lazy.Utils.shouldSkipRemoteActivityDueToTests) {
return;
}
diff --git a/toolkit/modules/AppConstants.sys.mjs b/toolkit/modules/AppConstants.sys.mjs
@@ -210,14 +210,16 @@ export var AppConstants = Object.freeze({
ENABLE_WEBDRIVER: @ENABLE_WEBDRIVER_BOOL@,
REMOTE_SETTINGS_SERVER_URL:
-#ifdef MOZ_THUNDERBIRD
+#if defined(BASE_BROWSER_VERSION)
+ "",
+#elif defined(MOZ_THUNDERBIRD)
"https://thunderbird-settings.thunderbird.net/v1",
#else
"https://firefox.settings.services.mozilla.com/v1",
#endif
REMOTE_SETTINGS_VERIFY_SIGNATURE:
-#ifdef MOZ_THUNDERBIRD
+#if defined(MOZ_THUNDERBIRD) || defined(BASE_BROWSER_VERSION)
false,
#else
true,
