tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

import-rollouts.js (12390B)

      1 /* This Source Code Form is subject to the terms of the Mozilla Public
      2 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
      3 * You can obtain one at http://mozilla.org/MPL/2.0/. */
      4 
      5 /**
      6 * This is a script to import Nimbus experiments from a given collection into
      7 * browser/components/asrouter/tests/NimbusRolloutMessageProvider.sys.mjs. By
      8 * default, it only imports messaging rollouts. This is done so that the content
      9 * of off-train rollouts can be easily searched. That way, when we are cleaning
     10 * up old assets (such as Fluent strings), we don't accidentally delete strings
     11 * that live rollouts are using because it was too difficult to find whether
     12 * they were in use.
     13 *
     14 * This works by fetching the message records from the Nimbus collection and
     15 * then writing them to the file. The messages are converted from JSON to JS.
     16 * The file is structured like this:
     17 * export const NimbusRolloutMessageProvider = {
     18 *   getMessages() {
     19 *     return [
     20 *       { ...message1 },
     21 *       { ...message2 },
     22 *     ];
     23 *   },
     24 * };
     25 */
     26 
     27 /* eslint-disable no-console */
     28 const chalk = require("chalk");
     29 const https = require("https");
     30 const path = require("path");
     31 const { pathToFileURL } = require("url");
     32 const fs = require("fs");
     33 const util = require("util");
     34 const prettier = require("prettier");
     35 const jsonschema = require("../../../../third_party/js/cfworker/json-schema.js");
     36 
     37 const DEFAULT_COLLECTION_ID = "nimbus-desktop-experiments";
     38 const BASE_URL =
     39  "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/";
     40 const EXPERIMENTER_URL = "https://experimenter.services.mozilla.com/nimbus/";
     41 const OUTPUT_PATH = "./tests/NimbusRolloutMessageProvider.sys.mjs";
     42 const LICENSE_STRING = `/* This Source Code Form is subject to the terms of the Mozilla Public
     43 * License, v. 2.0. If a copy of the MPL was not distributed with this
     44 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */`;
     45 
     46 function fetchJSON(url) {
     47  return new Promise((resolve, reject) => {
     48    https
     49      .get(url, resp => {
     50        let data = "";
     51        resp.on("data", chunk => {
     52          data += chunk;
     53        });
     54        resp.on("end", () => resolve(JSON.parse(data)));
     55      })
     56      .on("error", reject);
     57  });
     58 }
     59 
     60 function isMessageValid(validator, obj) {
     61  if (validator) {
     62    const result = validator.validate(obj);
     63    return result.valid && result.errors.length === 0;
     64  }
     65  return true;
     66 }
     67 
     68 async function getMessageValidators(skipValidation) {
     69  if (skipValidation) {
     70    return { experimentValidator: null, messageValidators: {} };
     71  }
     72 
     73  async function getSchema(filePath) {
     74    const file = await util.promisify(fs.readFile)(filePath, "utf8");
     75    return JSON.parse(file);
     76  }
     77 
     78  async function getValidator(filePath, { common = false } = {}) {
     79    const schema = await getSchema(filePath);
     80    const validator = new jsonschema.Validator(schema);
     81 
     82    if (common) {
     83      const commonSchema = await getSchema(
     84        "./content-src/schemas/FxMSCommon.schema.json"
     85      );
     86      validator.addSchema(commonSchema);
     87    }
     88 
     89    return validator;
     90  }
     91 
     92  const experimentValidator = await getValidator(
     93    "./content-src/schemas/MessagingExperiment.schema.json"
     94  );
     95 
     96  const messageValidators = {
     97    bookmarks_bar_button: await getValidator(
     98      "./content-src/templates/OnboardingMessage/BookmarksBarButton.schema.json",
     99      { common: true }
    100    ),
    101    cfr_doorhanger: await getValidator(
    102      "./content-src/templates/CFR/templates/ExtensionDoorhanger.schema.json",
    103      { common: true }
    104    ),
    105    cfr_urlbar_chiclet: await getValidator(
    106      "./content-src/templates/CFR/templates/CFRUrlbarChiclet.schema.json",
    107      { common: true }
    108    ),
    109    infobar: await getValidator(
    110      "./content-src/templates/CFR/templates/InfoBar.schema.json",
    111      { common: true }
    112    ),
    113    pb_newtab: await getValidator(
    114      "./content-src/templates/PBNewtab/NewtabPromoMessage.schema.json",
    115      { common: true }
    116    ),
    117    spotlight: await getValidator(
    118      "./content-src/templates/OnboardingMessage/Spotlight.schema.json",
    119      { common: true }
    120    ),
    121    toast_notification: await getValidator(
    122      "./content-src/templates/ToastNotification/ToastNotification.schema.json",
    123      { common: true }
    124    ),
    125    toolbar_badge: await getValidator(
    126      "./content-src/templates/OnboardingMessage/ToolbarBadgeMessage.schema.json",
    127      { common: true }
    128    ),
    129    update_action: await getValidator(
    130      "./content-src/templates/OnboardingMessage/UpdateAction.schema.json",
    131      { common: true }
    132    ),
    133    feature_callout: await getValidator(
    134      // For now, Feature Callout and Spotlight share a common schema
    135      "./content-src/templates/OnboardingMessage/Spotlight.schema.json",
    136      { common: true }
    137    ),
    138    menu_message: await getValidator(
    139      "./content-src/templates/OnboardingMessage/MenuMessage.schema.json",
    140      { common: true }
    141    ),
    142    newtab_message: await getValidator(
    143      "./content-src/templates/OnboardingMessage/NewtabMessage.schema.json",
    144      { common: true }
    145    ),
    146  };
    147 
    148  messageValidators.milestone_message = messageValidators.cfr_doorhanger;
    149 
    150  return { experimentValidator, messageValidators };
    151 }
    152 
    153 function annotateMessage({ message, slug, minVersion, maxVersion, url }) {
    154  const comments = [];
    155  if (slug) {
    156    comments.push(`// Nimbus slug: ${slug}`);
    157  }
    158  let versionRange = "";
    159  if (minVersion) {
    160    versionRange = minVersion;
    161    if (maxVersion) {
    162      versionRange += `-${maxVersion}`;
    163    } else {
    164      versionRange += "+";
    165    }
    166  } else if (maxVersion) {
    167    versionRange = `0-${maxVersion}`;
    168  }
    169  if (versionRange) {
    170    comments.push(`// Version range: ${versionRange}`);
    171  }
    172  if (url) {
    173    comments.push(`// Recipe: ${url}`);
    174  }
    175  return JSON.stringify(message, null, 2).replace(
    176    /^{/,
    177    `{ ${comments.join("\n")}`
    178  );
    179 }
    180 
    181 async function format(content) {
    182  const config = await prettier.resolveConfig("./.prettierrc.js");
    183  return prettier.format(content, { ...config, filepath: OUTPUT_PATH });
    184 }
    185 
    186 async function main() {
    187  const { default: meow } = await import("meow");
    188  const { MESSAGING_EXPERIMENTS_DEFAULT_FEATURES } = await import(
    189    "../modules/MessagingExperimentConstants.sys.mjs"
    190  );
    191 
    192  const fileUrl = pathToFileURL(__filename);
    193 
    194  const cli = meow(
    195    `
    196    Usage
    197      $ node bin/import-rollouts.js [options]
    198  
    199    Options
    200      -c ID, --collection ID   The Nimbus collection ID to import from
    201                               default: ${DEFAULT_COLLECTION_ID}
    202      -e, --experiments        Import all messaging experiments, not just rollouts
    203      -s, --skip-validation    Skip validation of experiments and messages
    204      -h, --help               Show this help message
    205  
    206    Examples
    207      $ node bin/import-rollouts.js --collection nimbus-preview
    208      $ ./mach npm run import-rollouts --prefix=browser/components/asrouter -- -e
    209  `,
    210    {
    211      description: false,
    212      // `pkg` is a tiny optimization. It prevents meow from looking for a package
    213      // that doesn't technically exist. meow searches for a package and changes
    214      // the process name to the package name. It resolves to the newtab
    215      // package.json, which would give a confusing name and be wasteful.
    216      pkg: {
    217        name: "import-rollouts",
    218        version: "1.0.0",
    219      },
    220      // `importMeta` is required by meow 10+. It was added to support ESM, but
    221      // meow now requires it, and no longer supports CJS style imports. But it
    222      // only uses import.meta.url, which can be polyfilled like this:
    223      importMeta: { url: fileUrl },
    224      flags: {
    225        collection: {
    226          type: "string",
    227          shortFlag: "c",
    228          default: DEFAULT_COLLECTION_ID,
    229        },
    230        experiments: {
    231          type: "boolean",
    232          shortFlag: "e",
    233          default: false,
    234        },
    235        skipValidation: {
    236          type: "boolean",
    237          shortFlag: "s",
    238          default: false,
    239        },
    240      },
    241    }
    242  );
    243 
    244  const RECORDS_URL = `${BASE_URL}${cli.flags.collection}/records`;
    245 
    246  console.log(`Fetching records from ${chalk.underline.yellow(RECORDS_URL)}`);
    247 
    248  const { data: records } = await fetchJSON(RECORDS_URL);
    249 
    250  if (!Array.isArray(records)) {
    251    throw new TypeError(
    252      `Expected records to be an array, got ${typeof records}`
    253    );
    254  }
    255 
    256  const recipes = records.filter(
    257    record =>
    258      record.application === "firefox-desktop" &&
    259      record.featureIds.some(id =>
    260        MESSAGING_EXPERIMENTS_DEFAULT_FEATURES.includes(id)
    261      ) &&
    262      (record.isRollout || cli.flags.experiments)
    263  );
    264 
    265  const importItems = [];
    266  const { experimentValidator, messageValidators } = await getMessageValidators(
    267    cli.flags.skipValidation
    268  );
    269  for (const recipe of recipes) {
    270    const { slug: experimentSlug, branches, targeting } = recipe;
    271    if (!(experimentSlug && Array.isArray(branches) && branches.length)) {
    272      continue;
    273    }
    274    console.log(
    275      `Processing ${recipe.isRollout ? "rollout" : "experiment"}: ${chalk.blue(
    276        experimentSlug
    277      )}${
    278        branches.length > 1
    279          ? ` with ${chalk.underline(`${String(branches.length)} branches`)}`
    280          : ""
    281      }`
    282    );
    283    const recipeUrl = `${EXPERIMENTER_URL}${experimentSlug}/summary`;
    284    const [, minVersion] =
    285      targeting?.match(/\(version\|versionCompare\(\'([0-9]+)\.!\'\) >= 0/) ||
    286      [];
    287    const [, maxVersion] =
    288      targeting?.match(/\(version\|versionCompare\(\'([0-9]+)\.\*\'\) <= 0/) ||
    289      [];
    290    let branchIndex = branches.length > 1 ? 1 : 0;
    291    for (const branch of branches) {
    292      const { slug: branchSlug, features } = branch;
    293      console.log(
    294        `  Processing branch${
    295          branchIndex > 0 ? ` ${branchIndex} of ${branches.length}` : ""
    296        }: ${chalk.blue(branchSlug)}`
    297      );
    298      branchIndex += 1;
    299      const url = `${recipeUrl}#${branchSlug}`;
    300      if (!Array.isArray(features)) {
    301        continue;
    302      }
    303      for (const feature of features) {
    304        if (
    305          feature.enabled &&
    306          MESSAGING_EXPERIMENTS_DEFAULT_FEATURES.includes(feature.featureId) &&
    307          feature.value &&
    308          typeof feature.value === "object" &&
    309          feature.value.template
    310        ) {
    311          if (!isMessageValid(experimentValidator, feature.value)) {
    312            console.log(
    313              `    ${chalk.red(
    314                "✗"
    315              )} Skipping invalid value for branch: ${chalk.blue(branchSlug)}`
    316            );
    317            continue;
    318          }
    319          const messages = (
    320            feature.value.template === "multi" &&
    321            Array.isArray(feature.value.messages)
    322              ? feature.value.messages
    323              : [feature.value]
    324          ).filter(m => m && m.id);
    325          let msgIndex = messages.length > 1 ? 1 : 0;
    326          for (const message of messages) {
    327            let messageLogString = `message${
    328              msgIndex > 0 ? ` ${msgIndex} of ${messages.length}` : ""
    329            }: ${chalk.italic.green(message.id)}`;
    330            if (!isMessageValid(messageValidators[message.template], message)) {
    331              console.log(
    332                `    ${chalk.red("✗")} Skipping invalid ${messageLogString}`
    333              );
    334              continue;
    335            }
    336            console.log(`    Importing ${messageLogString}`);
    337            let slug = `${experimentSlug}:${branchSlug}`;
    338            if (msgIndex > 0) {
    339              slug += ` (message ${msgIndex} of ${messages.length})`;
    340            }
    341            msgIndex += 1;
    342            importItems.push({ message, slug, minVersion, maxVersion, url });
    343          }
    344        }
    345      }
    346    }
    347  }
    348 
    349  const content = `${LICENSE_STRING}
    350 
    351 /**
    352 * This file is generated by browser/components/asrouter/bin/import-rollouts.js
    353 * Run the following from the repository root to regenerate it:
    354 * ./mach npm run import-rollouts --prefix=browser/components/asrouter
    355 */
    356 
    357 export const NimbusRolloutMessageProvider = {
    358  getMessages() {
    359    return [${importItems.map(annotateMessage).join(",\n")}];
    360  },
    361 };
    362 `;
    363 
    364  const formattedContent = await format(content);
    365 
    366  await util.promisify(fs.writeFile)(OUTPUT_PATH, formattedContent);
    367 
    368  console.log(
    369    `${chalk.green("✓")} Wrote ${chalk.underline.green(
    370      `${String(importItems.length)} ${
    371        importItems.length === 1 ? "message" : "messages"
    372      }`
    373    )} to ${chalk.underline.yellow(path.resolve(OUTPUT_PATH))}`
    374  );
    375 }
    376 
    377 main();