tor-browser

Utils.sys.mjs (18258B)

---

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs";
import { ServiceRequest } from "resource://gre/modules/ServiceRequest.sys.mjs";
import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";

const lazy = {};

ChromeUtils.defineESModuleGetters(lazy, {
 SharedUtils: "resource://services-settings/SharedUtils.sys.mjs",
});

XPCOMUtils.defineLazyServiceGetter(
 lazy,
 "CaptivePortalService",
 "@mozilla.org/network/captive-portal-service;1",
 Ci.nsICaptivePortalService
);
XPCOMUtils.defineLazyServiceGetter(
 lazy,
 "gNetworkLinkService",
 "@mozilla.org/network/network-link-service;1",
 Ci.nsINetworkLinkService
);

// Create a new instance of the ConsoleAPI so we can control the maxLogLevel with a pref.
// See LOG_LEVELS in Console.sys.mjs. Common examples: "all", "debug", "info",
// "warn", "error".
const log = (() => {
 const { ConsoleAPI } = ChromeUtils.importESModule(
   "resource://gre/modules/Console.sys.mjs"
 );
 return new ConsoleAPI({
   maxLogLevel: "warn",
   maxLogLevelPref: "services.settings.loglevel",
   prefix: "services.settings",
 });
})();

ChromeUtils.defineLazyGetter(lazy, "isRunningTests", () => {
 if (Services.env.get("MOZ_DISABLE_NONLOCAL_CONNECTIONS") === "1") {
   // Allow to override the server URL if non-local connections are disabled,
   // usually true when running tests.
   return true;
 }
 return false;
});

// Overriding the server URL is normally disabled on Beta and Release channels,
// except under some conditions.
ChromeUtils.defineLazyGetter(lazy, "allowServerURLOverride", () => {
 if (!AppConstants.RELEASE_OR_BETA) {
   // Always allow to override the server URL on Nightly/DevEdition.
   return true;
 }

 if (lazy.isRunningTests) {
   return true;
 }

 if (Services.env.get("MOZ_REMOTE_SETTINGS_DEVTOOLS") === "1") {
   // Allow to override the server URL when using remote settings devtools.
   return true;
 }

 if (lazy.gServerURL != AppConstants.REMOTE_SETTINGS_SERVER_URL) {
   log.warn("Ignoring preference override of remote settings server");
   log.warn(
     "Allow by setting MOZ_REMOTE_SETTINGS_DEVTOOLS=1 in the environment"
   );
 }

 return false;
});

XPCOMUtils.defineLazyPreferenceGetter(
 lazy,
 "gServerURL",
 "services.settings.server",
 AppConstants.REMOTE_SETTINGS_SERVER_URL
);

XPCOMUtils.defineLazyPreferenceGetter(
 lazy,
 "gPreviewEnabled",
 "services.settings.preview_enabled",
 false
);

function _isUndefined(value) {
 return typeof value === "undefined";
}

const _cdnURLs = {};

export var Utils = {
 get SERVER_URL() {
   return lazy.allowServerURLOverride
     ? lazy.gServerURL
     : AppConstants.REMOTE_SETTINGS_SERVER_URL;
 },

 CHANGES_PATH: "/buckets/monitor/collections/changes/changeset",

 /**
  * Logger instance.
  */
 log,

 get shouldSkipRemoteActivityDueToTests() {
   return (
     (lazy.isRunningTests || Cu.isInAutomation) &&
     this.SERVER_URL == "data:,#remote-settings-dummy/v1"
   );
 },

 get CERT_CHAIN_ROOT_IDENTIFIER() {
   if (Services.env.exists("XPCSHELL_TEST_PROFILE_DIR")) {
     return Ci.nsIX509CertDB.AppXPCShellRoot;
   }
   if (
     this.SERVER_URL.match(
       /^https?:\/\/(remote-settings\.localhost|127\.0\.0\.1|localhost)(:\d+)?\/v1/
     )
   ) {
     return Ci.nsIContentSignatureVerifier.ContentSignatureLocalRoot;
   }
   if (this.SERVER_URL.includes("allizom.")) {
     return Ci.nsIContentSignatureVerifier.ContentSignatureStageRoot;
   }
   if (this.SERVER_URL.includes("dev.")) {
     return Ci.nsIContentSignatureVerifier.ContentSignatureDevRoot;
   }
   return Ci.nsIContentSignatureVerifier.ContentSignatureProdRoot;
 },

 get LOAD_DUMPS() {
   // Load dumps only if pulling data from the production server, or in tests.
   return (
     this.SERVER_URL == AppConstants.REMOTE_SETTINGS_SERVER_URL ||
     lazy.isRunningTests
   );
 },

 get PREVIEW_MODE() {
   // We want to offer the ability to set preview mode via a preference
   // for consumers who want to pull from the preview bucket on startup.
   if (_isUndefined(this._previewModeEnabled) && lazy.allowServerURLOverride) {
     return lazy.gPreviewEnabled;
   }
   return !!this._previewModeEnabled;
 },

 /**
  * Internal method to enable pulling data from preview buckets.
  *
  * @param enabled
  */
 enablePreviewMode(enabled) {
   const bool2str = v =>
     // eslint-disable-next-line no-nested-ternary
     _isUndefined(v) ? "unset" : v ? "enabled" : "disabled";
   this.log.debug(
     `Preview mode: ${bool2str(this._previewModeEnabled)} -> ${bool2str(
       enabled
     )}`
   );
   this._previewModeEnabled = enabled;
 },

 /**
  * Returns the actual bucket name to be used. When preview mode is enabled,
  * this adds the *preview* suffix.
  *
  * See also `SharedUtils.loadJSONDump()` which strips the preview suffix to identify
  * the packaged JSON file.
  *
  * @param bucketName the client bucket
  * @returns the final client bucket depending whether preview mode is enabled.
  */
 actualBucketName(bucketName) {
   let actual = bucketName.replace("-preview", "");
   if (this.PREVIEW_MODE) {
     actual += "-preview";
   }
   return actual;
 },

 /**
  * Check if network is down.
  *
  * Note that if this returns false, it does not guarantee
  * that network is up.
  *
  * @return {bool} Whether network is down or not.
  */
 get isOffline() {
   try {
     return (
       Services.io.offline ||
       lazy.CaptivePortalService.state ==
         lazy.CaptivePortalService.LOCKED_PORTAL ||
       !lazy.gNetworkLinkService.isLinkUp
     );
   } catch (ex) {
     log.warn("Could not determine network status.", ex);
   }
   return false;
 },

 /**
  * A wrapper around `ServiceRequest` that behaves like `fetch()`.
  *
  * Use this in order to leverage the `beConservative` flag, for
  * example to avoid using HTTP3 to fetch critical data.
  *
  * @param input a resource
  * @param init request options
  * @returns a Response object
  */
 async fetch(input, init = {}) {
   return new Promise(function (resolve, reject) {
     const request = new ServiceRequest();
     function fallbackOrReject(err) {
       if (
         // At most one recursive Utils.fetch call (bypassProxy=false to true).
         bypassProxy ||
         Services.startup.shuttingDown ||
         Utils.isOffline ||
         !request.isProxied ||
         !request.bypassProxyEnabled
       ) {
         reject(err);
         return;
       }
       ServiceRequest.logProxySource(request.channel, "remote-settings");
       resolve(Utils.fetch(input, { ...init, bypassProxy: true }));
     }

     request.onerror = () =>
       fallbackOrReject(new TypeError("NetworkError: Network request failed"));
     request.ontimeout = () =>
       fallbackOrReject(new TypeError("Timeout: Network request failed"));
     request.onabort = () =>
       fallbackOrReject(new DOMException("Aborted", "AbortError"));
     request.onload = () => {
       // Parse raw response headers into `Headers` object.
       const headers = new Headers();
       const rawHeaders = request.getAllResponseHeaders();
       rawHeaders
         .trim()
         .split(/[\r\n]+/)
         .forEach(line => {
           const parts = line.split(": ");
           const header = parts.shift();
           const value = parts.join(": ");
           headers.set(header, value);
         });

       const responseAttributes = {
         status: request.status,
         statusText: request.statusText,
         url: request.responseURL,
         headers,
       };
       resolve(new Response(request.response, responseAttributes));
     };

     const { method = "GET", headers = {}, bypassProxy = false } = init;

     request.open(method, input, { bypassProxy });
     // By default, XMLHttpRequest converts the response based on the
     // Content-Type header, or UTF-8 otherwise. This may mangle binary
     // responses. Avoid that by requesting the raw bytes.
     request.responseType = "arraybuffer";

     for (const [name, value] of Object.entries(headers)) {
       request.setRequestHeader(name, value);
     }

     request.send();
   });
 },

 /**
  * Retrieves the base URL for attachments from the server configuration.
  *
  * If the URL has been previously fetched and cached, it returns the cached URL.
  *
  * @async
  * @function baseAttachmentsURL
  * @memberof Utils
  * @returns {Promise<string>} A promise that resolves to the base URL for attachments.
  *
  * @throws {Error} If there is an error fetching or parsing the server response.
  *
  * @example
  * const attachmentsURL = await Downloader.baseAttachmentsURL();
  * console.log(attachmentsURL);
  */
 async baseAttachmentsURL() {
   if (!_cdnURLs[Utils.SERVER_URL]) {
     const resp = await Utils.fetch(`${Utils.SERVER_URL}/`);
     const serverInfo = await resp.json();
     // Server capabilities expose attachments configuration.
     const {
       capabilities: {
         attachments: { base_url },
       },
     } = serverInfo;
     // Make sure the URL always has a trailing slash.
     _cdnURLs[Utils.SERVER_URL] =
       base_url + (base_url.endsWith("/") ? "" : "/");
   }
   return _cdnURLs[Utils.SERVER_URL];
 },

 /**
  * Check if local data exist for the specified client.
  *
  * @param {RemoteSettingsClient} client
  * @return {bool} Whether it exists or not.
  */
 async hasLocalData(client) {
   const timestamp = await client.db.getLastModified();
   return timestamp !== null;
 },

 /**
  * Check if we ship a JSON dump for the specified bucket and collection.
  *
  * @param {string} bucket
  * @param {string} collection
  * @return {bool} Whether it is present or not.
  */
 async hasLocalDump(bucket, collection) {
   try {
     await fetch(
       `resource://app/defaults/settings/${bucket}/${collection}.json`,
       {
         method: "HEAD",
       }
     );
     return true;
   } catch (e) {
     return false;
   }
 },

 /**
  * Look up the last modification time of the JSON dump.
  *
  * @param {string} bucket
  * @param {string} collection
  * @return {int} The last modification time of the dump. -1 if non-existent.
  */
 async getLocalDumpLastModified(bucket, collection) {
   if (!this._dumpStats) {
     if (!this._dumpStatsInitPromise) {
       this._dumpStatsInitPromise = (async () => {
         try {
           let res = await fetch(
             "resource://app/defaults/settings/last_modified.json"
           );
           this._dumpStats = await res.json();
         } catch (e) {
           log.warn(`Failed to load last_modified.json: ${e}`);
           this._dumpStats = {};
         }
         delete this._dumpStatsInitPromise;
       })();
     }
     await this._dumpStatsInitPromise;
   }
   const identifier = `${bucket}/${collection}`;
   let lastModified = this._dumpStats[identifier];
   if (lastModified === undefined) {
     const { timestamp: dumpTimestamp } = await lazy.SharedUtils.loadJSONDump(
       bucket,
       collection
     );
     // Client recognize -1 as missing dump.
     lastModified = dumpTimestamp ?? -1;
     this._dumpStats[identifier] = lastModified;
   }
   return lastModified;
 },

 /**
  * Fetch the list of remote collections and their timestamp.
  * ```
  *   {
  *     "timestamp": 1486545678,
  *     "changes":[
  *       {
  *         "host":"kinto-ota.dev.mozaws.net",
  *         "last_modified":1450717104423,
  *         "bucket":"blocklists",
  *         "collection":"certificates"
  *       },
  *       ...
  *     ],
  *     "metadata": {}
  *   }
  * ```
  *
  * @param {string} serverUrl         The server URL (eg. `https://server.org/v1`)
  * @param {int}    expectedTimestamp The timestamp that the server is supposed to return.
  *                                   We obtained it from the Megaphone notification payload,
  *                                   and we use it only for cache busting (Bug 1497159).
  * @param {string} lastEtag          (optional) The Etag of the latest poll to be matched
  *                                   by the server (eg. `"123456789"`).
  * @param {object} filters
  */
 async fetchLatestChanges(serverUrl, options = {}) {
   const { expectedTimestamp, lastEtag = "", filters = {} } = options;

   let url = serverUrl + Utils.CHANGES_PATH;
   const params = {
     ...filters,
     _expected: expectedTimestamp ?? 0,
   };
   if (lastEtag != "") {
     params._since = lastEtag;
   }
   if (params) {
     url +=
       "?" +
       Object.entries(params)
         .map(([k, v]) => `${k}=${encodeURIComponent(v)}`)
         .join("&");
   }
   const response = await Utils.fetch(url);

   if (response.status >= 500) {
     throw new Error(`Server error ${response.status} ${response.statusText}`);
   }

   const is404FromCustomServer =
     response.status == 404 &&
     Services.prefs.prefHasUserValue("services.settings.server");

   const ct = response.headers.get("Content-Type");
   if (!is404FromCustomServer && (!ct || !ct.includes("application/json"))) {
     throw new Error(`Unexpected content-type "${ct}"`);
   }

   let payload;
   try {
     payload = await response.json();
   } catch (e) {
     payload = e.message;
   }

   if (!payload.hasOwnProperty("changes")) {
     // If the server is failing, the JSON response might not contain the
     // expected data. For example, real server errors (Bug 1259145)
     // or dummy local server for tests (Bug 1481348)
     if (!is404FromCustomServer) {
       throw new Error(
         `Server error ${url} ${response.status} ${
           response.statusText
         }: ${JSON.stringify(payload)}`
       );
     }
   }

   const { changes = [], timestamp } = payload;

   let serverTimeMillis = Date.parse(response.headers.get("Date"));
   // Since the response is served via a CDN, the Date header value could have been cached.
   const cacheAgeSeconds = response.headers.has("Age")
     ? parseInt(response.headers.get("Age"), 10)
     : 0;
   serverTimeMillis += cacheAgeSeconds * 1000;

   // Age of data (time between publication and now).
   const ageSeconds = (serverTimeMillis - timestamp) / 1000;

   // Check if the server asked the clients to back off.
   let backoffSeconds;
   if (response.headers.has("Backoff")) {
     const value = parseInt(response.headers.get("Backoff"), 10);
     if (!isNaN(value)) {
       backoffSeconds = value;
     }
   }

   return {
     changes,
     currentEtag: `"${timestamp}"`,
     serverTimeMillis,
     backoffSeconds,
     ageSeconds,
   };
 },

 /**
  * Test if a single object matches all given filters.
  *
  * @param  {object} filters  The filters object.
  * @param  {object} entry    The object to filter.
  * @return {boolean}
  */
 filterObject(filters, entry) {
   return Object.entries(filters).every(([filter, value]) => {
     if (Array.isArray(value)) {
       return value.some(candidate => candidate === entry[filter]);
     } else if (typeof value === "object") {
       return Utils.filterObject(value, entry[filter]);
     } else if (!Object.prototype.hasOwnProperty.call(entry, filter)) {
       log.debug(`The property ${filter} does not exist`);
       return false;
     }
     return entry[filter] === value;
   });
 },

 /**
  * Sorts records in a list according to a given ordering.
  *
  * @param  {string} order The ordering, eg. `-last_modified`.
  * @param  {Array}  list  The collection to order.
  * @return {Array}
  */
 sortObjects(order, list) {
   const hasDash = order[0] === "-";
   const field = hasDash ? order.slice(1) : order;
   const direction = hasDash ? -1 : 1;
   return list.slice().sort((a, b) => {
     if (a[field] && _isUndefined(b[field])) {
       return direction;
     }
     if (b[field] && _isUndefined(a[field])) {
       return -direction;
     }
     if (_isUndefined(a[field]) && _isUndefined(b[field])) {
       return 0;
     }
     return a[field] > b[field] ? direction : -direction;
   });
 },

 /**
  * Fetches and extracts a bundle of changesets from the server.
  *
  * This function downloads a JSON file with all changesets required during startup, compressed as LZ4.
  * It writes the LZ4 file to a temporary location, extracts it, and return the array of changesets.
  * We chose to use LZ4 instead of Zip because extraction can happen off the main thread.
  *
  * @async
  * @function fetchChangesetsBundle
  * @memberof Utils
  * @returns {Promise<Array<object>>} A promise that resolves to an array of parsed changesets.
  *
  * @throws {Error} Throws an error if there is an issue fetching the server info or the changeset bundle,
  *                 or if there is an error during the extraction and parsing of the changesets.
  */
 async fetchChangesetsBundle() {
   const tmpLz4File = await IOUtils.createUniqueFile(
     PathUtils.tempDir,
     "remote-settings-startup-bundle-"
   );
   try {
     const baseUrl = await Utils.baseAttachmentsURL();
     const bundleUrl = `${baseUrl}bundles/startup.json.mozlz4`;
     const bundleResp = await Utils.fetch(bundleUrl);
     if (!bundleResp.ok) {
       throw new Error(`Cannot fetch changeset bundle from ${bundleUrl}`);
     }
     // Write down the LZ4 in a temporary file.
     const downloaded = await bundleResp.arrayBuffer();
     await IOUtils.write(tmpLz4File, new Uint8Array(downloaded), {
       tmpPath: `${tmpLz4File}.tmp`,
     });
     // Decompress using LZ4
     const changesetsJson = await IOUtils.readUTF8(tmpLz4File, {
       decompress: true,
     });
     // Parse JSON from string
     return JSON.parse(changesetsJson);
   } finally {
     await IOUtils.remove(tmpLz4File, { ignoreAbsent: true });
   }
 },
};