tor-browser

DiscoveryStreamFeed.sys.mjs (100128B)

---

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
 ContextId: "moz-src:///browser/modules/ContextId.sys.mjs",
 DEFAULT_SECTION_LAYOUT: "resource://newtab/lib/SectionsLayoutManager.sys.mjs",
 NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs",
 NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
 ObliviousHTTP: "resource://gre/modules/ObliviousHTTP.sys.mjs",
 PersistentCache: "resource://newtab/lib/PersistentCache.sys.mjs",
 Region: "resource://gre/modules/Region.sys.mjs",
 RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
 ProfileAge: "resource://gre/modules/ProfileAge.sys.mjs",
});

// We use importESModule here instead of static import so that
// the Karma test environment won't choke on this module. This
// is because the Karma test environment already stubs out
// setTimeout / clearTimeout, and overrides importESModule
// to be a no-op (which can't be done for a static import statement).

// eslint-disable-next-line mozilla/use-static-import
const { setTimeout, clearTimeout } = ChromeUtils.importESModule(
 "resource://gre/modules/Timer.sys.mjs"
);
import {
 actionTypes as at,
 actionCreators as ac,
} from "resource://newtab/common/Actions.mjs";

import { scoreItemInferred } from "resource://newtab/lib/InferredModel/GreedyContentRanker.mjs";

const LOCAL_POPULAR_RERANK = false; // default behavior for local re-ranking
const LOCAL_WEIGHT = 1;
const SERVER_WEIGHT = 1;
const CACHE_KEY = "discovery_stream";
const STARTUP_CACHE_EXPIRE_TIME = 7 * 24 * 60 * 60 * 1000; // 1 week
const COMPONENT_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
const SPOCS_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
const DEFAULT_RECS_ROTATION_TIME = 60 * 60 * 1000; // 1 hour
const DEFAULT_RECS_IMPRESSION_EXPIRE_TIME = 7 * 24 * 60 * 60 * 1000; // 7 days
const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server
const SPOCS_CAP_DURATION = 24 * 60 * 60; // 1 day in seconds.
const FETCH_TIMEOUT = 45 * 1000;
const TOPIC_LOADING_TIMEOUT = 1 * 1000;
const TOPIC_SELECTION_DISPLAY_COUNT =
 "discoverystream.topicSelection.onboarding.displayCount";
const TOPIC_SELECTION_LAST_DISPLAYED =
 "discoverystream.topicSelection.onboarding.lastDisplayed";
const TOPIC_SELECTION_DISPLAY_TIMEOUT =
 "discoverystream.topicSelection.onboarding.displayTimeout";

const SPOCS_URL = "https://spocs.getpocket.com/spocs";
const PREF_CONFIG = "discoverystream.config";
const PREF_ENDPOINTS = "discoverystream.endpoints";
const PREF_IMPRESSION_ID = "browser.newtabpage.activity-stream.impressionId";
// const PREF_LAYOUT_EXPERIMENT_A = "newtabLayouts.variant-a";
// const PREF_LAYOUT_EXPERIMENT_B = "newtabLayouts.variant-b";
const PREF_CONTEXTUAL_SPOC_PLACEMENTS =
 "discoverystream.placements.contextualSpocs";
const PREF_CONTEXTUAL_SPOC_COUNTS =
 "discoverystream.placements.contextualSpocs.counts";
const PREF_SPOC_PLACEMENTS = "discoverystream.placements.spocs";
const PREF_SPOC_COUNTS = "discoverystream.placements.spocs.counts";
const PREF_SPOC_POSITIONS = "discoverystream.spoc-positions";
const PREF_MERINO_FEED_EXPERIMENT =
 "browser.newtabpage.activity-stream.discoverystream.merino-feed-experiment";
const PREF_ENABLED = "discoverystream.enabled";
const PREF_HARDCODED_BASIC_LAYOUT = "discoverystream.hardcoded-basic-layout";
const PREF_SPOCS_ENDPOINT = "discoverystream.spocs-endpoint";
const PREF_SPOCS_ENDPOINT_QUERY = "discoverystream.spocs-endpoint-query";
const PREF_REGION_BASIC_LAYOUT = "discoverystream.region-basic-layout";
const PREF_USER_TOPSTORIES = "feeds.section.topstories";
const PREF_SYSTEM_TOPSTORIES = "feeds.system.topstories";
const PREF_UNIFIED_ADS_BLOCKED_LIST = "unifiedAds.blockedAds";
const PREF_UNIFIED_ADS_SPOCS_ENABLED = "unifiedAds.spocs.enabled";
const PREF_UNIFIED_ADS_ADSFEED_ENABLED = "unifiedAds.adsFeed.enabled";
const PREF_UNIFIED_ADS_ENDPOINT = "unifiedAds.endpoint";
const PREF_UNIFIED_ADS_OHTTP = "unifiedAds.ohttp.enabled";
const PREF_SPOCS_CLEAR_ENDPOINT = "discoverystream.endpointSpocsClear";
const PREF_SHOW_SPONSORED = "showSponsored";
const PREF_SYSTEM_SHOW_SPONSORED = "system.showSponsored";
const PREF_SPOC_IMPRESSIONS = "discoverystream.spoc.impressions";
const PREF_FLIGHT_BLOCKS = "discoverystream.flight.blocks";
const PREF_SELECTED_TOPICS = "discoverystream.topicSelection.selectedTopics";
const PREF_TOPIC_SELECTION_ENABLED = "discoverystream.topicSelection.enabled";
const PREF_TOPIC_SELECTION_PREVIOUS_SELECTED =
 "discoverystream.topicSelection.hasBeenUpdatedPreviously";
const PREF_SPOCS_CACHE_ONDEMAND = "discoverystream.spocs.onDemand";
const PREF_SPOCS_CACHE_TIMEOUT = "discoverystream.spocs.cacheTimeout";
const PREF_SPOCS_STARTUP_CACHE_ENABLED =
 "discoverystream.spocs.startupCache.enabled";
const PREF_CONTEXTUAL_ADS = "discoverystream.sections.contextualAds.enabled";
const PREF_USER_INFERRED_PERSONALIZATION =
 "discoverystream.sections.personalization.inferred.user.enabled";
const PREF_SYSTEM_INFERRED_PERSONALIZATION =
 "discoverystream.sections.personalization.inferred.enabled";
const PREF_INFERRED_INTERESTS_OVERRIDE =
 "discoverystream.sections.personalization.inferred.interests.override";

const PREF_MERINO_OHTTP = "discoverystream.merino-provider.ohttp.enabled";
const PREF_BILLBOARD_ENABLED = "newtabAdSize.billboard";
const PREF_LEADERBOARD_ENABLED = "newtabAdSize.leaderboard";
const PREF_LEADERBOARD_POSITION = "newtabAdSize.leaderboard.position";
const PREF_BILLBOARD_POSITION = "newtabAdSize.billboard.position";
const PREF_CONTEXTUAL_BANNER_PLACEMENTS =
 "discoverystream.placements.contextualBanners";
const PREF_CONTEXTUAL_BANNER_COUNTS =
 "discoverystream.placements.contextualBanners.counts";

const PREF_SECTIONS_ENABLED = "discoverystream.sections.enabled";
const PREF_SECTIONS_FOLLOWING = "discoverystream.sections.following";
const PREF_SECTIONS_BLOCKED = "discoverystream.sections.blocked";
const PREF_INTEREST_PICKER_ENABLED =
 "discoverystream.sections.interestPicker.enabled";
const PREF_VISIBLE_SECTIONS =
 "discoverystream.sections.interestPicker.visibleSections";
const PREF_PRIVATE_PING_ENABLED = "telemetry.privatePing.enabled";
const PREF_SURFACE_ID = "telemetry.surfaceId";
const PREF_CLIENT_LAYOUT_ENABLED =
 "discoverystream.sections.clientLayout.enabled";

let getHardcodedLayout;

ChromeUtils.defineLazyGetter(lazy, "userAgent", () => {
 return Cc["@mozilla.org/network/protocol;1?name=http"].getService(
   Ci.nsIHttpProtocolHandler
 ).userAgent;
});

export class DiscoveryStreamFeed {
 constructor() {
   // Internal state for checking if we've intialized all our data
   this.loaded = false;

   // Persistent cache for remote endpoint data.
   this.cache = new lazy.PersistentCache(CACHE_KEY, true);
   this.locale = Services.locale.appLocaleAsBCP47;
   this._impressionId = this.getOrCreateImpressionId();
   // Internal in-memory cache for parsing json prefs.
   this._prefCache = {};
 }

 getOrCreateImpressionId() {
   let impressionId = Services.prefs.getCharPref(PREF_IMPRESSION_ID, "");
   if (!impressionId) {
     impressionId = String(Services.uuid.generateUUID());
     Services.prefs.setCharPref(PREF_IMPRESSION_ID, impressionId);
   }
   return impressionId;
 }

 get config() {
   if (this._prefCache.config) {
     return this._prefCache.config;
   }
   try {
     this._prefCache.config = JSON.parse(
       this.store.getState().Prefs.values[PREF_CONFIG]
     );
   } catch (e) {
     // istanbul ignore next
     this._prefCache.config = {};
     // istanbul ignore next
     console.error(
       `Could not parse preference. Try resetting ${PREF_CONFIG} in about:config.`,
       e
     );
   }
   this._prefCache.config.enabled =
     this._prefCache.config.enabled &&
     this.store.getState().Prefs.values[PREF_ENABLED];

   return this._prefCache.config;
 }

 resetConfigDefauts() {
   this.store.dispatch({
     type: at.CLEAR_PREF,
     data: {
       name: PREF_CONFIG,
     },
   });
 }

 get region() {
   return lazy.Region.home;
 }

 get isContextualAds() {
   if (this._isContextualAds === undefined) {
     // We care about if the contextual ads pref is on, if contextual is supported,
     // and if inferred is on, but OHTTP is off.
     const state = this.store.getState();
     const marsOhttpEnabled = state.Prefs.values[PREF_UNIFIED_ADS_OHTTP];
     const contextualAds = state.Prefs.values[PREF_CONTEXTUAL_ADS];
     const inferredPersonalization =
       state.Prefs.values[PREF_USER_INFERRED_PERSONALIZATION] &&
       state.Prefs.values[PREF_SYSTEM_INFERRED_PERSONALIZATION];
     const sectionsEnabled = state.Prefs.values[PREF_SECTIONS_ENABLED];
     // We want this if contextual ads are on, and also if inferred personalization is on, we also use OHTTP.
     const useContextualAds =
       contextualAds &&
       ((inferredPersonalization && marsOhttpEnabled) ||
         !inferredPersonalization);
     this._isContextualAds = sectionsEnabled && useContextualAds;
   }

   return this._isContextualAds;
 }

 get doLocalInferredRerank() {
   if (this._doLocalInferredRerank === undefined) {
     const state = this.store.getState();

     const inferredPersonalization =
       state.Prefs.values[PREF_USER_INFERRED_PERSONALIZATION] &&
       state.Prefs.values[PREF_SYSTEM_INFERRED_PERSONALIZATION];
     const sectionsEnabled = state.Prefs.values[PREF_SECTIONS_ENABLED];

     const systemPref = inferredPersonalization && sectionsEnabled;
     const expPref =
       state.Prefs.values.inferredPersonalizationConfig
         ?.local_popular_today_rerank ?? LOCAL_POPULAR_RERANK;

     // we do it if inferred is on and the experiment is on
     this._doLocalInferredRerank = systemPref && expPref;
   }
   return this._doLocalInferredRerank;
 }

 get showSponsoredStories() {
   // Combine user-set sponsored opt-out with Mozilla-set config
   return (
     this.store.getState().Prefs.values[PREF_SHOW_SPONSORED] &&
     this.store.getState().Prefs.values[PREF_SYSTEM_SHOW_SPONSORED]
   );
 }

 get showStories() {
   // Combine user-set stories opt-out with Mozilla-set config
   return (
     this.store.getState().Prefs.values[PREF_SYSTEM_TOPSTORIES] &&
     this.store.getState().Prefs.values[PREF_USER_TOPSTORIES]
   );
 }

 get personalized() {
   return this.recommendationProvider.personalized;
 }

 get recommendationProvider() {
   if (this._recommendationProvider) {
     return this._recommendationProvider;
   }
   this._recommendationProvider = this.store.feeds.get(
     "feeds.recommendationprovider"
   );
   return this._recommendationProvider;
 }

 setupConfig(isStartup = false) {
   // Send the initial state of the pref on our reducer
   this.store.dispatch(
     ac.BroadcastToContent({
       type: at.DISCOVERY_STREAM_CONFIG_SETUP,
       data: this.config,
       meta: {
         isStartup,
       },
     })
   );
 }

 async setupDevtoolsState(isStartup = false) {
   const cachedData = (await this.cache.get()) || {};
   let impressions = cachedData.recsImpressions || {};
   let blocks = cachedData.recsBlocks || {};

   this.store.dispatch({
     type: at.DISCOVERY_STREAM_DEV_IMPRESSIONS,
     data: impressions,
     meta: {
       isStartup,
     },
   });

   this.store.dispatch({
     type: at.DISCOVERY_STREAM_DEV_BLOCKS,
     data: blocks,
     meta: {
       isStartup,
     },
   });
 }

 setupPrefs(isStartup = false) {
   const experimentMetadata =
     lazy.NimbusFeatures.pocketNewtab.getEnrollmentMetadata();

   let utmSource = "pocket-newtab";
   let utmCampaign = experimentMetadata?.slug;
   let utmContent = experimentMetadata?.branch;

   this.store.dispatch(
     ac.BroadcastToContent({
       type: at.DISCOVERY_STREAM_EXPERIMENT_DATA,
       data: {
         utmSource,
         utmCampaign,
         utmContent,
       },
       meta: {
         isStartup,
       },
     })
   );

   const nimbusConfig = this.store.getState().Prefs.values?.pocketConfig || {};
   const { region } = this.store.getState().Prefs.values;

   const hideDescriptionsRegions = nimbusConfig.hideDescriptionsRegions
     ?.split(",")
     .map(s => s.trim());
   const hideDescriptions =
     nimbusConfig.hideDescriptions ||
     hideDescriptionsRegions?.includes(region);

   // We don't BroadcastToContent for this, as the changes may
   // shift around elements on an open newtab the user is currently reading.
   // So instead we AlsoToPreloaded so the next tab is updated.
   // This is because setupPrefs is called by the system and not a user interaction.
   this.store.dispatch(
     ac.AlsoToPreloaded({
       type: at.DISCOVERY_STREAM_PREFS_SETUP,
       data: {
         hideDescriptions,
         compactImages: nimbusConfig.compactImages,
         imageGradient: nimbusConfig.imageGradient,
         newSponsoredLabel: nimbusConfig.newSponsoredLabel,
         titleLines: nimbusConfig.titleLines,
         descLines: nimbusConfig.descLines,
         readTime: nimbusConfig.readTime,
       },
       meta: {
         isStartup,
       },
     })
   );

   // sync redux store with PersistantCache personalization data
   this.configureFollowedSections(isStartup);
 }

 async configureFollowedSections(isStartup = false) {
   const prefs = this.store.getState().Prefs.values;
   const cachedData = (await this.cache.get()) || {};
   let { sectionPersonalization } = cachedData;

   // if sectionPersonalization is empty, populate it with data from the followed and blocked prefs
   // eventually we could remove this (maybe once more of sections is added to release)
   if (
     sectionPersonalization &&
     Object.keys(sectionPersonalization).length === 0
   ) {
     // Raw string of followed/blocked topics, ex: "entertainment, news"
     const followedSectionsString = prefs[PREF_SECTIONS_FOLLOWING];
     const blockedSectionsString = prefs[PREF_SECTIONS_BLOCKED];
     // Format followed sections
     const followedSections = followedSectionsString
       ? followedSectionsString.split(",").map(s => s.trim())
       : [];

     // Format blocked sections
     const blockedSections = blockedSectionsString
       ? blockedSectionsString.split(",").map(s => s.trim())
       : [];

     const sectionTopics = new Set([...followedSections, ...blockedSections]);
     sectionPersonalization = Array.from(sectionTopics).reduce(
       (acc, section) => {
         acc[section] = {
           isFollowed: followedSections.includes(section),
           isBlocked: blockedSections.includes(section),
         };
         return acc;
       },
       {}
     );
     await this.cache.set(
       "sectionPersonalization",
       sectionPersonalization || {}
     );
   }
   this.store.dispatch(
     ac.BroadcastToContent({
       type: at.SECTION_PERSONALIZATION_UPDATE,
       data: sectionPersonalization || {},
       meta: {
         isStartup,
       },
     })
   );
 }

 uninitPrefs() {
   // Reset in-memory cache
   this._prefCache = {};
 }

 async fetchFromEndpoint(endpoint, options = {}, useOhttp = false) {
   let fetchPromise;
   if (!endpoint) {
     console.error("Tried to fetch endpoint but none was configured.");
     return null;
   }

   const ohttpRelayURL = Services.prefs.getStringPref(
     "browser.newtabpage.activity-stream.discoverystream.ohttp.relayURL",
     ""
   );
   const ohttpConfigURL = Services.prefs.getStringPref(
     "browser.newtabpage.activity-stream.discoverystream.ohttp.configURL",
     ""
   );

   try {
     // Make sure the requested endpoint is allowed
     const allowed =
       this.store
         .getState()
         .Prefs.values[PREF_ENDPOINTS].split(",")
         .map(item => item.trim())
         .filter(item => item) || [];
     if (!allowed.some(prefix => endpoint.startsWith(prefix))) {
       throw new Error(`Not one of allowed prefixes (${allowed})`);
     }

     const controller = new AbortController();
     const { signal } = controller;

     if (useOhttp && ohttpConfigURL && ohttpRelayURL) {
       let config = await lazy.ObliviousHTTP.getOHTTPConfig(ohttpConfigURL);

       if (!config) {
         console.error(
           new Error(
             `OHTTP was configured for ${endpoint} but we couldn't fetch a valid config`
           )
         );
         return null;
       }

       // ObliviousHTTP.ohttpRequest only accepts a key/value object, and not
       // a Headers instance. We normalize any headers to a key/value object.
       //
       // We use instanceof here since isInstance isn't available for node
       // tests like DiscoveryStreamFeed.test.js.
       // eslint-disable-next-line mozilla/use-isInstance
       if (options.headers && options.headers instanceof Headers) {
         options.headers = Object.fromEntries(options.headers);
       }

       fetchPromise = lazy.ObliviousHTTP.ohttpRequest(
         ohttpRelayURL,
         config,
         endpoint,
         {
           ...options,
           credentials: "omit",
           signal,
         }
       );
     } else {
       fetchPromise = fetch(endpoint, {
         ...options,
         credentials: "omit",
         signal,
       });
     }

     // istanbul ignore next
     const timeoutId = setTimeout(() => {
       controller.abort();
     }, FETCH_TIMEOUT);

     const response = await fetchPromise;

     if (!response.ok) {
       throw new Error(`Unexpected status (${response.status})`);
     }
     clearTimeout(timeoutId);

     return response.json();
   } catch (error) {
     console.error(`Failed to fetch ${endpoint}:`, error.message);
   }
   return null;
 }
 get spocsOnDemand() {
   if (this._spocsOnDemand === undefined) {
     const { values } = this.store.getState().Prefs;
     const spocsOnDemandConfig = values.trainhopConfig?.spocsOnDemand || {};
     const spocsOnDemand =
       spocsOnDemandConfig.enabled || values[PREF_SPOCS_CACHE_ONDEMAND];
     this._spocsOnDemand = this.showSponsoredStories && spocsOnDemand;
   }

   return this._spocsOnDemand;
 }

 get spocsCacheUpdateTime() {
   if (this._spocsCacheUpdateTime === undefined) {
     const { values } = this.store.getState().Prefs;
     const spocsOnDemandConfig = values.trainhopConfig?.spocsOnDemand || {};
     const spocsCacheTimeout =
       spocsOnDemandConfig.timeout || values[PREF_SPOCS_CACHE_TIMEOUT];
     const MAX_TIMEOUT = 30;
     const MIN_TIMEOUT = 5;

     // We have some guard rails against misconfigured values.
     // Ignore 0: a zero-minute timeout would cause constant fetches.
     // Check min max times, or ensure we don't make requests on a timer.
     const guardRailed =
       spocsCacheTimeout &&
       (this.spocsOnDemand ||
         (spocsCacheTimeout <= MAX_TIMEOUT &&
           spocsCacheTimeout >= MIN_TIMEOUT));

     if (guardRailed) {
       // This value is in minutes, but we want ms.
       this._spocsCacheUpdateTime = spocsCacheTimeout * 60 * 1000;
     } else {
       // The const is already in ms.
       this._spocsCacheUpdateTime = SPOCS_FEEDS_UPDATE_TIME;
     }
   }

   return this._spocsCacheUpdateTime;
 }

 /**
  * Returns true if data in the cache for a particular key has expired or is missing.
  *
  * @param {object} cachedData data returned from cache.get()
  * @param {string} key a cache key
  * @param {string?} url for "feed" only, the URL of the feed.
  * @param {boolean} is this check done at initial browser load
  */
 isExpired({ cachedData, key, url, isStartup }) {
   const { spocs, feeds } = cachedData;
   const updateTimePerComponent = {
     spocs: this.spocsCacheUpdateTime,
     feed: COMPONENT_FEEDS_UPDATE_TIME,
   };
   const EXPIRATION_TIME = isStartup
     ? STARTUP_CACHE_EXPIRE_TIME
     : updateTimePerComponent[key];

   switch (key) {
     case "spocs":
       return !spocs || !(Date.now() - spocs.lastUpdated < EXPIRATION_TIME);
     case "feed": {
       if (!feeds || !feeds[url]) {
         return true;
       }
       const feed = feeds[url];
       const isTimeExpired = Date.now() - feed.lastUpdated >= EXPIRATION_TIME;
       const sectionsEnabled =
         this.store.getState().Prefs.values[PREF_SECTIONS_ENABLED];
       const sectionsEnabledChanged = feed.sectionsEnabled !== sectionsEnabled;
       return isTimeExpired || sectionsEnabledChanged;
     }
     default:
       // istanbul ignore next
       throw new Error(`${key} is not a valid key`);
   }
 }

 async _checkExpirationPerComponent() {
   const cachedData = (await this.cache.get()) || {};
   const { feeds } = cachedData;

   return {
     spocs:
       this.showSponsoredStories &&
       this.isExpired({ cachedData, key: "spocs" }),
     feeds:
       this.showStories &&
       (!feeds ||
         Object.keys(feeds).some(url =>
           this.isExpired({ cachedData, key: "feed", url })
         )),
   };
 }

 updatePlacements(sendUpdate, layout, isStartup = false) {
   const placements = [];
   const placementsMap = {};
   for (const row of layout.filter(r => r.components && r.components.length)) {
     for (const component of row.components.filter(
       c => c.placement && c.spocs
     )) {
       // If we find a valid placement, we set it to this value.
       let placement;

       if (this.showSponsoredStories) {
         placement = component.placement;
       }

       // Validate this placement and check for dupes.
       if (placement?.name && !placementsMap[placement.name]) {
         placementsMap[placement.name] = placement;
         placements.push(placement);
       }
     }
   }

   // Update placements data.
   // Even if we have no placements, we still want to update it to clear it.
   sendUpdate({
     type: at.DISCOVERY_STREAM_SPOCS_PLACEMENTS,
     data: { placements },
     meta: {
       isStartup,
     },
   });
 }

 /**
  * Adds a query string to a URL.
  * A query can be any string literal accepted by https://developer.mozilla.org/docs/Web/API/URLSearchParams
  * Examples: "?foo=1&bar=2", "&foo=1&bar=2", "foo=1&bar=2", "?bar=2" or "bar=2"
  */
 addEndpointQuery(url, query) {
   if (!query) {
     return url;
   }

   const urlObject = new URL(url);
   const params = new URLSearchParams(query);

   for (let [key, val] of params.entries()) {
     urlObject.searchParams.append(key, val);
   }

   return urlObject.toString();
 }

 parseGridPositions(csvPositions) {
   let gridPositions;

   // Only accept parseable non-negative integers
   try {
     gridPositions = csvPositions.map(index => {
       let parsedInt = parseInt(index, 10);

       if (!isNaN(parsedInt) && parsedInt >= 0) {
         return parsedInt;
       }

       throw new Error("Bad input");
     });
   } catch (e) {
     // Catch spoc positions that are not numbers or negative, and do nothing.
     // We have hard coded backup positions.
     gridPositions = undefined;
   }

   return gridPositions;
 }

 generateFeedUrl() {
   return `https://${Services.prefs.getStringPref(
     "browser.newtabpage.activity-stream.discoverystream.merino-provider.endpoint"
   )}/api/v1/curated-recommendations`;
 }

 loadLayout(sendUpdate, isStartup) {
   let layoutData = {};
   let url = "";

   const isBasicLayout =
     this.config.hardcoded_basic_layout ||
     this.store.getState().Prefs.values[PREF_HARDCODED_BASIC_LAYOUT] ||
     this.store.getState().Prefs.values[PREF_REGION_BASIC_LAYOUT];

   const pocketConfig = this.store.getState().Prefs.values?.pocketConfig || {};
   const items = isBasicLayout ? 4 : 24;
   const ctaButtonSponsors = pocketConfig.ctaButtonSponsors
     ?.split(",")
     .map(s => s.trim().toLowerCase());
   let ctaButtonVariant = "";
   // We specifically against hard coded values, instead of applying whatever is in the pref.
   // This is to ensure random class names from a user modified pref doesn't make it into the class list.
   if (
     pocketConfig.ctaButtonVariant === "variant-a" ||
     pocketConfig.ctaButtonVariant === "variant-b"
   ) {
     ctaButtonVariant = pocketConfig.ctaButtonVariant;
   }

   const topicSelectionHasBeenUpdatedPreviously =
     this.store.getState().Prefs.values[
       PREF_TOPIC_SELECTION_PREVIOUS_SELECTED
     ];

   const selectedTopics =
     this.store.getState().Prefs.values[PREF_SELECTED_TOPICS];

   // Note: This requires a cache update to react to a pref update
   const pocketStoriesHeadlineId =
     topicSelectionHasBeenUpdatedPreviously || selectedTopics
       ? "newtab-section-header-todays-picks"
       : "newtab-section-header-stories";

   pocketConfig.pocketStoriesHeadlineId = pocketStoriesHeadlineId;

   const prepConfArr = arr => {
     return arr
       ?.split(",")
       .filter(item => item)
       .map(item => parseInt(item, 10));
   };

   const spocAdTypes = prepConfArr(pocketConfig.spocAdTypes);
   const spocZoneIds = prepConfArr(pocketConfig.spocZoneIds);
   const { spocSiteId } = pocketConfig;
   let spocPlacementData;
   let spocsUrl;

   if (spocAdTypes?.length && spocZoneIds?.length) {
     spocPlacementData = {
       ad_types: spocAdTypes,
       zone_ids: spocZoneIds,
     };
   }

   if (spocSiteId) {
     const newUrl = new URL(SPOCS_URL);
     newUrl.searchParams.set("site", spocSiteId);
     spocsUrl = newUrl.href;
   }

   let feedUrl = this.generateFeedUrl();

   // Set layout config.
   // Changing values in this layout in memory object is unnecessary.
   layoutData = getHardcodedLayout({
     spocsUrl,
     feedUrl,
     items,
     spocPlacementData,
     spocPositions: this.parseGridPositions(
       this.store.getState().Prefs.values[PREF_SPOC_POSITIONS]?.split(`,`)
     ),
     widgetPositions: this.parseGridPositions(
       pocketConfig.widgetPositions?.split(`,`)
     ),
     widgetData: [
       ...(this.locale.startsWith("en-") ? [{ type: "TopicsWidget" }] : []),
     ],
     hybridLayout: pocketConfig.hybridLayout,
     hideCardBackground: pocketConfig.hideCardBackground,
     fourCardLayout: pocketConfig.fourCardLayout,
     newFooterSection: pocketConfig.newFooterSection,
     compactGrid: pocketConfig.compactGrid,
     // For now button variants are for experimentation and English only.
     ctaButtonSponsors: this.locale.startsWith("en-") ? ctaButtonSponsors : [],
     ctaButtonVariant: this.locale.startsWith("en-") ? ctaButtonVariant : "",
     pocketStoriesHeadlineId: pocketConfig.pocketStoriesHeadlineId,
   });

   sendUpdate({
     type: at.DISCOVERY_STREAM_LAYOUT_UPDATE,
     data: layoutData,
     meta: {
       isStartup,
     },
   });

   if (layoutData.spocs) {
     url =
       this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT] ||
       this.config.spocs_endpoint ||
       layoutData.spocs.url;

     const spocsEndpointQuery =
       this.store.getState().Prefs.values[PREF_SPOCS_ENDPOINT_QUERY];

     // For QA, testing, or debugging purposes, there may be a query string to add.
     url = this.addEndpointQuery(url, spocsEndpointQuery);

     if (
       url &&
       url !== this.store.getState().DiscoveryStream.spocs.spocs_endpoint
     ) {
       sendUpdate({
         type: at.DISCOVERY_STREAM_SPOCS_ENDPOINT,
         data: {
           url,
         },
         meta: {
           isStartup,
         },
       });
       this.updatePlacements(sendUpdate, layoutData.layout, isStartup);
     }
   }
 }

 /**
  * Adds the promise result to newFeeds and pushes a promise to newsFeedsPromises.
  *
  * @param {object} Has both newFeedsPromises (Array) and newFeeds (Object)
  * @param {boolean} isStartup We have different cache handling for startup.
  * @returns {Function} We return a function so we can contain
  *                     the scope for isStartup and the promises object.
  *                     Combines feed results and promises for each component with a feed.
  */
 buildFeedPromise(
   { newFeedsPromises, newFeeds },
   isStartup = false,
   sendUpdate
 ) {
   return component => {
     const { url } = component.feed;

     if (!newFeeds[url]) {
       // We initially stub this out so we don't fetch dupes,
       // we then fill in with the proper object inside the promise.
       newFeeds[url] = {};
       const feedPromise = this.getComponentFeed(url, isStartup);

       feedPromise
         .then(feed => {
           // I think we could reduce doing this for cache fetches.
           // Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606277
           // We can remove filterRecommendations once ESR catches up to bug 1932196
           newFeeds[url] = this.filterRecommendations(feed);
           sendUpdate({
             type: at.DISCOVERY_STREAM_FEED_UPDATE,
             data: {
               feed: newFeeds[url],
               url,
             },
             meta: {
               isStartup,
             },
           });
         })
         .catch(
           /* istanbul ignore next */ error => {
             console.error(
               `Error trying to load component feed ${url}:`,
               error
             );
           }
         );
       newFeedsPromises.push(feedPromise);
     }
   };
 }

 // This filters just recommendations using NewTabUtils.blockedLinks only.
 // This is essentially a sync blocked links filter. filterBlocked is async.
 // See bug 1606277.
 filterRecommendations(feed) {
   if (feed?.data?.recommendations?.length) {
     const recommendations = feed.data.recommendations.filter(item => {
       const blocked = lazy.NewTabUtils.blockedLinks.isBlocked({
         url: item.url,
       });
       return !blocked;
     });

     return {
       ...feed,
       data: {
         ...feed.data,
         recommendations,
       },
     };
   }
   return feed;
 }

 /**
  * Filters out components with no feeds, and combines all feeds on this component
  * with the feeds from other components.
  *
  * @param {boolean} isStartup We have different cache handling for startup.
  * @returns {Function} We return a function so we can contain the scope for isStartup.
  *                     Reduces feeds into promises and feed data.
  */
 reduceFeedComponents(isStartup, sendUpdate) {
   return (accumulator, row) => {
     row.components
       .filter(component => component && component.feed)
       .forEach(this.buildFeedPromise(accumulator, isStartup, sendUpdate));
     return accumulator;
   };
 }

 /**
  * Filters out rows with no components, and gets us a promise for each unique feed.
  *
  * @param {object} layout This is the Discovery Stream layout object.
  * @param {boolean} isStartup We have different cache handling for startup.
  * @returns {object} An object with newFeedsPromises (Array) and newFeeds (Object),
  *                   we can Promise.all newFeedsPromises to get completed data in newFeeds.
  */
 buildFeedPromises(layout, isStartup, sendUpdate) {
   const initialData = {
     newFeedsPromises: [],
     newFeeds: {},
   };
   return layout
     .filter(row => row && row.components)
     .reduce(this.reduceFeedComponents(isStartup, sendUpdate), initialData);
 }

 async loadComponentFeeds(sendUpdate, isStartup = false) {
   const { DiscoveryStream } = this.store.getState();

   if (!DiscoveryStream || !DiscoveryStream.layout) {
     return;
   }

   // Reset the flag that indicates whether or not at least one API request
   // was issued to fetch the component feed in `getComponentFeed()`.
   this.componentFeedFetched = false;
   const { newFeedsPromises, newFeeds } = this.buildFeedPromises(
     DiscoveryStream.layout,
     isStartup,
     sendUpdate
   );

   // Each promise has a catch already built in, so no need to catch here.
   await Promise.all(newFeedsPromises);
   await this.cache.set("feeds", newFeeds);
   sendUpdate({
     type: at.DISCOVERY_STREAM_FEEDS_UPDATE,
     meta: {
       isStartup,
     },
   });
 }

 getPlacements() {
   const { placements } = this.store.getState().DiscoveryStream.spocs;
   return placements;
 }

 // I wonder, can this be better as a reducer?
 // See Bug https://bugzilla.mozilla.org/show_bug.cgi?id=1606717
 placementsForEach(callback) {
   this.getPlacements().forEach(callback);
 }

 // Bug 1567271 introduced meta data on a list of spocs.
 // This involved moving the spocs array into an items prop.
 // However, old data could still be returned, and cached data might also be old.
 // For ths reason, we want to ensure if we don't find an items array,
 // we use the previous array placement, and then stub out title and context to empty strings.
 // We need to do this *after* both fresh fetches and cached data to reduce repetition.

 // Bug 1916488 introduced a new data stricture from the unified ads API.
 // We want to maintain both implementations until we're done rollout out,
 // so for now we are going to normlaize the new data to match the old data props,
 // so we can change as little as possible. Once we commit to one, we can remove all this.
 normalizeSpocsItems(spocs) {
   const unifiedAdsEnabled =
     this.store.getState().Prefs.values[PREF_UNIFIED_ADS_SPOCS_ENABLED];
   if (unifiedAdsEnabled) {
     return {
       items: spocs.map(spoc => ({
         format: spoc.format,
         alt_text: spoc.alt_text,
         id: spoc.caps?.cap_key,
         flight_id: spoc.block_key,
         block_key: spoc.block_key,
         shim: spoc.callbacks,
         caps: {
           flight: {
             count: spoc.caps?.day,
             period: SPOCS_CAP_DURATION,
           },
         },
         domain: spoc.domain,
         excerpt: spoc.excerpt,
         raw_image_src: spoc.image_url,
         priority: spoc.ranking?.priority || 1,
         personalization_models: spoc.ranking?.personalization_models,
         item_score: spoc.ranking?.item_score,
         sponsor: spoc.sponsor,
         title: spoc.title,
         url: spoc.url,
         attribution: spoc.attributions || null,
       })),
     };
   }

   const items = spocs.items || spocs;
   const title = spocs.title || "";
   const context = spocs.context || "";
   const sponsor = spocs.sponsor || "";
   // We do not stub sponsored_by_override with an empty string. It is an override, and an empty string
   // explicitly means to override the client to display an empty string.
   // An empty string is not an no op in this case. Undefined is the proper no op here.
   const { sponsored_by_override } = spocs;
   // Undefined is fine here. It's optional and only used by collections.
   // If we leave it out, you get a collection that cannot be dismissed.
   const { flight_id } = spocs;
   return {
     items,
     title,
     context,
     sponsor,
     sponsored_by_override,
     ...(flight_id ? { flight_id } : {}),
   };
 }

 // This returns ad placements that contain IAB content.
 // The results are ads that are contextual, and match an IAB category.
 getContextualAdsPlacements() {
   const state = this.store.getState();

   const billboardEnabled = state.Prefs.values[PREF_BILLBOARD_ENABLED];
   const billboardPosition = state.Prefs.values[PREF_BILLBOARD_POSITION];
   const leaderboardEnabled = state.Prefs.values[PREF_LEADERBOARD_ENABLED];
   const leaderboardPosition = state.Prefs.values[PREF_LEADERBOARD_POSITION];

   function getContextualStringPref(prefName) {
     return state.Prefs.values[prefName]
       ?.split(",")
       .map(s => s.trim())
       .filter(item => item);
   }

   function getContextualCountPref(prefName) {
     return state.Prefs.values[prefName]
       ?.split(`,`)
       .map(s => s.trim())
       .filter(item => item)
       .map(item => parseInt(item, 10));
   }

   const placementSpocsArray = getContextualStringPref(
     PREF_CONTEXTUAL_SPOC_PLACEMENTS
   );
   const countsSpocsArray = getContextualCountPref(
     PREF_CONTEXTUAL_SPOC_COUNTS
   );
   const bannerPlacementsArray = getContextualStringPref(
     PREF_CONTEXTUAL_BANNER_PLACEMENTS
   );
   const bannerCountsArray = getContextualCountPref(
     PREF_CONTEXTUAL_BANNER_COUNTS
   );

   const feeds = state.DiscoveryStream.feeds.data;

   const recsFeed = Object.values(feeds).find(
     feed => feed?.data?.sections?.length
   );

   let iabSections = [];
   let iabPlacements = [];
   let bannerPlacements = [];

   // If we don't have recsFeed, it means we are loading for the first time,
   // and don't have any cached data.
   // In this situation, we don't fill iabPlacements,
   // and go with the non IAB default contextual placement prefs.
   if (recsFeed) {
     iabSections = recsFeed.data.sections
       .filter(section => section.iab)
       .sort((a, b) => a.receivedRank - b.receivedRank);

     // An array of all iab placement, flattened, sorted, and filtered.
     iabPlacements = iabSections
       // .filter(section => section.iab)
       // .sort((a, b) => a.receivedRank - b.receivedRank)
       .reduce((acc, section) => {
         const iabArray = section.layout.responsiveLayouts[0].tiles
           .filter(tile => tile.hasAd)
           .map(() => {
             return section.iab;
           });
         return [...acc, ...iabArray];
       }, []);
   }

   const spocPlacements = placementSpocsArray.map((placement, index) => ({
     placement,
     count: countsSpocsArray[index],
     ...(iabPlacements[index] ? { content: iabPlacements[index] } : {}),
   }));

   if (billboardEnabled) {
     bannerPlacements = bannerPlacementsArray.map((placement, index) => ({
       placement,
       count: bannerCountsArray[index],
       ...(iabSections[billboardPosition - 2]
         ? { content: iabSections[billboardPosition - 2].iab }
         : {}),
     }));
   } else if (leaderboardEnabled) {
     bannerPlacements = bannerPlacementsArray.map((placement, index) => ({
       placement,
       count: bannerCountsArray[index],
       ...(iabSections[leaderboardPosition - 2]
         ? { content: iabSections[leaderboardPosition - 2].iab }
         : {}),
     }));
   }

   return [...spocPlacements, ...bannerPlacements];
 }

 // This returns ad placements that don't contain IAB content.
 // The results are ads that are not contextual, and can be of any IAB category.
 getSimpleAdsPlacements() {
   const state = this.store.getState();
   const placementsArray = state.Prefs.values[PREF_SPOC_PLACEMENTS]?.split(`,`)
     .map(s => s.trim())
     .filter(item => item);
   const countsArray = state.Prefs.values[PREF_SPOC_COUNTS]?.split(`,`)
     .map(s => s.trim())
     .filter(item => item)
     .map(item => parseInt(item, 10));

   return placementsArray.map((placement, index) => ({
     placement,
     count: countsArray[index],
   }));
 }

 getAdsPlacements() {
   // We can replace unifiedAdsPlacements if we have and can use contextual ads.
   // No longer relying on pref based placements and counts.
   if (this.isContextualAds) {
     return this.getContextualAdsPlacements();
   }
   return this.getSimpleAdsPlacements();
 }

 async updateOrRemoveSpocs() {
   const dispatch = update =>
     this.store.dispatch(ac.BroadcastToContent(update));
   // We refresh placements data because one of the spocs were turned off.
   this.updatePlacements(
     dispatch,
     this.store.getState().DiscoveryStream.layout
   );
   // Currently the order of this is important.
   // We need to check this after updatePlacements is called,
   // because some of the spoc logic depends on the result of placement updates.
   if (!this.showSponsoredStories) {
     // Ensure we delete any remote data potentially related to spocs.
     this.clearSpocs();
   }
   // Placements have changed so consider spocs expired, and reload them.
   await this.cache.set("spocs", {});
   await this.loadSpocs(dispatch);
 }

 // eslint-disable-next-line max-statements
 async loadSpocs(sendUpdate, isStartup) {
   const cachedData = (await this.cache.get()) || {};
   const unifiedAdsEnabled =
     this.store.getState().Prefs.values[PREF_UNIFIED_ADS_SPOCS_ENABLED];

   const adsFeedEnabled =
     this.store.getState().Prefs.values[PREF_UNIFIED_ADS_ADSFEED_ENABLED];

   let spocsState = cachedData.spocs;
   let placements = this.getPlacements();
   let unifiedAdsPlacements = [];

   if (
     this.showSponsoredStories &&
     placements?.length &&
     this.isExpired({ cachedData, key: "spocs", isStartup })
   ) {
     if (placements?.length) {
       const headers = new Headers();
       headers.append("content-type", "application/json");
       const apiKeyPref = this.config.api_key_pref;
       const apiKey = Services.prefs.getCharPref(apiKeyPref, "");
       const state = this.store.getState();
       let endpoint = state.DiscoveryStream.spocs.spocs_endpoint;
       let body = {
         pocket_id: this._impressionId,
         version: 2,
         consumer_key: apiKey,
         ...(placements.length ? { placements } : {}),
       };

       const marsOhttpEnabled = state.Prefs.values[PREF_UNIFIED_ADS_OHTTP];

       // Bug 1964715: Remove this logic when AdsFeed is 100% enabled
       if (unifiedAdsEnabled && !adsFeedEnabled) {
         const endpointBaseUrl = state.Prefs.values[PREF_UNIFIED_ADS_ENDPOINT];
         endpoint = `${endpointBaseUrl}v1/ads`;
         unifiedAdsPlacements = this.getAdsPlacements();
         const blockedSponsors =
           state.Prefs.values[PREF_UNIFIED_ADS_BLOCKED_LIST];

         // We need some basic data that we can pass along to the ohttp request.
         // We purposefully don't use ohttp on this request. We also expect to
         // mostly hit the HTTP cache rather than the network with these requests.
         if (marsOhttpEnabled) {
           const preFlight = await this.fetchFromEndpoint(
             `${endpointBaseUrl}v1/ads-preflight`,
             {
               method: "GET",
             }
           );

           if (preFlight) {
             // If we don't get a normalized_ua, it means it matched the default userAgent.
             headers.append(
               "X-User-Agent",
               preFlight.normalized_ua || lazy.userAgent
             );
             headers.append("X-Geoname-ID", preFlight.geoname_id);
             headers.append("X-Geo-Location", preFlight.geo_location);
           }
         }

         body = {
           context_id: await lazy.ContextId.request(),
           placements: unifiedAdsPlacements,
           blocks: blockedSponsors.split(","),
         };
       }

       let spocsResponse;
       // Logic decision point: Query ads servers in this file or utilize AdsFeed method
       if (adsFeedEnabled) {
         const { spocs, spocPlacements } = state.Ads;

         if (spocs) {
           spocsResponse = { newtab_spocs: spocs };
           unifiedAdsPlacements = spocPlacements;
         } else {
           throw new Error("DSFeed cannot read AdsFeed spocs");
         }
       } else {
         try {
           spocsResponse = await this.fetchFromEndpoint(
             endpoint,
             {
               method: "POST",
               headers,
               body: JSON.stringify(body),
             },
             marsOhttpEnabled
           );
         } catch (error) {
           console.error("Error trying to load spocs feeds:", error);
         }
       }

       if (spocsResponse) {
         const fetchTimestamp = Date.now();
         spocsState = {
           lastUpdated: fetchTimestamp,
           spocs: {
             ...spocsResponse,
           },
         };

         if (spocsResponse.settings && spocsResponse.settings.feature_flags) {
           this.store.dispatch(
             ac.OnlyToMain({
               type: at.DISCOVERY_STREAM_PERSONALIZATION_OVERRIDE,
               data: {
                 override: !spocsResponse.settings.feature_flags.spoc_v2,
               },
             })
           );
         }

         const spocsResultPromises = this.getPlacements().map(
           async placement => {
             let freshSpocs = spocsState.spocs[placement.name];

             if (unifiedAdsEnabled) {
               if (!unifiedAdsPlacements) {
                 throw new Error("unifiedAdsPlacements has no value");
               }

               // No placements to reduce upon
               if (!unifiedAdsPlacements.length) {
                 return;
               }

               freshSpocs = unifiedAdsPlacements.reduce(
                 (accumulator, currentValue) => {
                   return accumulator.concat(
                     spocsState.spocs[currentValue.placement]
                   );
                 },
                 []
               );
             }

             if (!freshSpocs) {
               return;
             }

             // spocs can be returns as an array, or an object with an items array.
             // We want to normalize this so all our spocs have an items array.
             // There can also be some meta data for title and context.
             // This is mostly because of backwards compat.
             const {
               items: normalizedSpocsItems,
               title,
               context,
               sponsor,
               sponsored_by_override,
             } = this.normalizeSpocsItems(freshSpocs);

             if (!normalizedSpocsItems || !normalizedSpocsItems.length) {
               // In the case of old data, we still want to ensure we normalize the data structure,
               // even if it's empty. We expect the empty data to be an object with items array,
               // and not just an empty array.
               spocsState.spocs = {
                 ...spocsState.spocs,
                 [placement.name]: {
                   title,
                   context,
                   items: [],
                 },
               };
               return;
             }

             // Migrate flight_id
             const { data: migratedSpocs } =
               this.migrateFlightId(normalizedSpocsItems);

             const { data: capResult } = this.frequencyCapSpocs(migratedSpocs);

             const { data: blockedResults } =
               await this.filterBlocked(capResult);

             const { data: spocsWithFetchTimestamp } = this.addFetchTimestamp(
               blockedResults,
               fetchTimestamp
             );

             let items = spocsWithFetchTimestamp;
             let personalized = false;

             // We only need to rank if we don't have contextual ads.
             if (!this.isContextualAds) {
               const scoreResults = await this.scoreItems(
                 spocsWithFetchTimestamp,
                 "spocs"
               );
               items = scoreResults.data;
               personalized = scoreResults.personalized;
             }

             spocsState.spocs = {
               ...spocsState.spocs,
               [placement.name]: {
                 title,
                 context,
                 sponsor,
                 sponsored_by_override,
                 personalized,
                 items,
               },
             };
           }
         );
         await Promise.all(spocsResultPromises);

         this.cleanUpFlightImpressionPref(spocsState.spocs);
       } else {
         console.error("No response for spocs_endpoint prop");
       }
     }
   }

   // Use good data if we have it, otherwise nothing.
   // We can have no data if spocs set to off.
   // We can have no data if request fails and there is no good cache.
   // We want to send an update spocs or not, so client can render something.
   spocsState =
     spocsState && spocsState.spocs
       ? spocsState
       : {
           lastUpdated: Date.now(),
           spocs: {},
         };
   await this.cache.set("spocs", {
     lastUpdated: spocsState.lastUpdated,
     spocs: spocsState.spocs,
     spocsOnDemand: this.spocsOnDemand,
     spocsCacheUpdateTime: this.spocsCacheUpdateTime,
   });

   sendUpdate({
     type: at.DISCOVERY_STREAM_SPOCS_UPDATE,
     data: {
       lastUpdated: spocsState.lastUpdated,
       spocs: spocsState.spocs,
       spocsOnDemand: this.spocsOnDemand,
       spocsCacheUpdateTime: this.spocsCacheUpdateTime,
     },
     meta: {
       isStartup,
     },
   });
 }

 async clearSpocs() {
   const state = this.store.getState();
   let endpoint = state.Prefs.values[PREF_SPOCS_CLEAR_ENDPOINT];

   const unifiedAdsEnabled =
     state.Prefs.values[PREF_UNIFIED_ADS_SPOCS_ENABLED];

   let body = {
     pocket_id: this._impressionId,
   };

   if (unifiedAdsEnabled) {
     const adsFeedEnabled =
       state.Prefs.values[PREF_UNIFIED_ADS_ADSFEED_ENABLED];

     const endpointBaseUrl = state.Prefs.values[PREF_UNIFIED_ADS_ENDPOINT];

     // Exit if there no DELETE endpoint or AdsFeed is enabled (which will handle the DELETE request)
     if (!endpointBaseUrl || adsFeedEnabled) {
       return;
     }

     // If rotation is enabled, then the module is going to take care of
     // sending the request to MARS to delete the context_id. Otherwise,
     // we do it manually here.
     if (lazy.ContextId.rotationEnabled) {
       await lazy.ContextId.forceRotation();
     } else {
       endpoint = `${endpointBaseUrl}v1/delete_user`;
       body = {
         context_id: await lazy.ContextId.request(),
       };
     }
   }

   if (!endpoint) {
     return;
   }
   const headers = new Headers();
   headers.append("content-type", "application/json");
   const marsOhttpEnabled = state.Prefs.values[PREF_UNIFIED_ADS_OHTTP];

   await this.fetchFromEndpoint(
     endpoint,
     {
       method: "DELETE",
       headers,
       body: JSON.stringify(body),
     },
     marsOhttpEnabled
   );
 }

 /*
  * This function is used to sort any type of story, both spocs and recs.
  * This uses hierarchical sorting, first sorting by priority, then by score within a priority.
  * This function could be sorting an array of spocs or an array of recs.
  * A rec would have priority undefined, and a spoc would probably have a priority set.
  * Priority is sorted ascending, so low numbers are the highest priority.
  * Score is sorted descending, so high numbers are the highest score.
  * Undefined priority values are considered the lowest priority.
  * A negative priority is considered the same as undefined, lowest priority.
  * A negative priority is unlikely and not currently supported or expected.
  * A negative score is a possible use case.
  */
 sortItem(a, b) {
   // If the priorities are the same, sort based on score.
   // If both item priorities are undefined,
   // we can safely sort via score.
   if (a.priority === b.priority) {
     return b.score - a.score;
   } else if (!a.priority || a.priority <= 0) {
     // If priority is undefined or an unexpected value,
     // consider it lowest priority.
     return 1;
   } else if (!b.priority || b.priority <= 0) {
     // Also consider this case lowest priority.
     return -1;
   }
   // Our primary sort for items with priority.
   return a.priority - b.priority;
 }

 async scoreItems(items, type) {
   const spocsPersonalized =
     this.store.getState().Prefs.values?.pocketConfig?.spocsPersonalized;
   const recsPersonalized =
     this.store.getState().Prefs.values?.pocketConfig?.recsPersonalized;
   const personalizedByType =
     type === "feed" ? recsPersonalized : spocsPersonalized;
   // If this is initialized, we are ready to go.
   let personalized = this.store.getState().Personalization.initialized;
   let data = null;
   if (type === "feed" && this.doLocalInferredRerank) {
     // make a flag for this
     const { inferredInterests = {} } =
       this.store.getState().InferredPersonalization ?? {};
     const weights = {
       inferred_norm: Object.entries(inferredInterests).reduce(
         (acc, [, v]) =>
           Number.isFinite(v) && !Number.isInteger(v) ? acc + v : acc,
         0
       ),
       local:
         (this.store.getState().Prefs.values?.inferredPersonalizationConfig
           ?.local_inferred_weight ?? LOCAL_WEIGHT) / 100,
       server:
         (this.store.getState().Prefs.values?.inferredPersonalizationConfig
           ?.server_inferred_weight ?? SERVER_WEIGHT) / 100,
     };
     data = (
       await Promise.all(
         items.map(item => scoreItemInferred(item, inferredInterests, weights))
       )
     )
       // Sort by highest scores.
       .sort(this.sortItem);
     personalized = true;
   } else {
     data = (
       await Promise.all(
         items.map(item => this.scoreItem(item, personalizedByType))
       )
     )
       // Sort by highest scores.
       .sort(this.sortItem);
   }

   return { data, personalized };
 }

 async scoreItem(item, personalizedByType) {
   item.score = item.item_score;
   if (item.score !== 0 && !item.score) {
     item.score = 1;
   }
   if (this.personalized && personalizedByType) {
     await this.recommendationProvider.calculateItemRelevanceScore(item);
   }
   return item;
 }

 async filterBlocked(data) {
   if (data?.length) {
     let flights = this.readDataPref(PREF_FLIGHT_BLOCKS);

     const cachedData = (await this.cache.get()) || {};
     let blocks = cachedData.recsBlocks || {};

     const filteredItems = data.filter(item => {
       const blocked =
         lazy.NewTabUtils.blockedLinks.isBlocked({ url: item.url }) ||
         flights[item.flight_id] ||
         blocks[item.id];
       return !blocked;
     });
     return { data: filteredItems };
   }
   return { data };
 }

 // Add the fetch timestamp property to each spoc returned to communicate how
 // old the spoc is in telemetry when it is used by the client
 addFetchTimestamp(spocs, fetchTimestamp) {
   if (spocs && spocs.length) {
     return {
       data: spocs.map(s => {
         return {
           ...s,
           fetchTimestamp,
         };
       }),
     };
   }
   return { data: spocs };
 }

 // For backwards compatibility, older spoc endpoint don't have flight_id,
 // but instead had campaign_id we can use
 //
 // @param {Object} data  An object that might have a SPOCS array.
 // @returns {Object} An object with a property `data` as the result.
 migrateFlightId(spocs) {
   if (spocs && spocs.length) {
     return {
       data: spocs.map(s => {
         return {
           ...s,
           ...(s.flight_id || s.campaign_id
             ? {
                 flight_id: s.flight_id || s.campaign_id,
               }
             : {}),
           ...(s.caps
             ? {
                 caps: {
                   ...s.caps,
                   flight: s.caps.flight || s.caps.campaign,
                 },
               }
             : {}),
         };
       }),
     };
   }
   return { data: spocs };
 }

 // Filter spocs based on frequency caps
 //
 // @param {Object} data  An object that might have a SPOCS array.
 // @returns {Object} An object with a property `data` as the result, and a property
 //                   `filterItems` as the frequency capped items.
 frequencyCapSpocs(spocs) {
   if (spocs?.length) {
     const impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS);
     const caps = [];
     const result = spocs.filter(s => {
       const isBelow = this.isBelowFrequencyCap(impressions, s);
       if (!isBelow) {
         caps.push(s);
       }
       return isBelow;
     });
     // send caps to redux if any.
     if (caps.length) {
       this.store.dispatch({
         type: at.DISCOVERY_STREAM_SPOCS_CAPS,
         data: caps,
       });
     }
     return { data: result, filtered: caps };
   }
   return { data: spocs, filtered: [] };
 }

 // Frequency caps are based on flight, which may include multiple spocs.
 // We currently support two types of frequency caps:
 // - lifetime: Indicates how many times spocs from a flight can be shown in total
 // - period: Indicates how many times spocs from a flight can be shown within a period
 //
 // So, for example, the feed configuration below defines that for flight 1 no more
 // than 5 spocs can be shown in total, and no more than 2 per hour.
 // "flight_id": 1,
 // "caps": {
 //  "lifetime": 5,
 //  "flight": {
 //    "count": 2,
 //    "period": 3600
 //  }
 // }
 isBelowFrequencyCap(impressions, spoc) {
   const flightImpressions = impressions[spoc.flight_id];
   if (!flightImpressions) {
     return true;
   }

   const lifetime = spoc.caps && spoc.caps.lifetime;

   const lifeTimeCap = Math.min(
     lifetime || MAX_LIFETIME_CAP,
     MAX_LIFETIME_CAP
   );
   const lifeTimeCapExceeded = flightImpressions.length >= lifeTimeCap;
   if (lifeTimeCapExceeded) {
     return false;
   }

   const flightCap = spoc.caps && spoc.caps.flight;
   if (flightCap) {
     const flightCapExceeded =
       flightImpressions.filter(i => Date.now() - i < flightCap.period * 1000)
         .length >= flightCap.count;
     return !flightCapExceeded;
   }
   return true;
 }

 async retryFeed(feed) {
   const { url } = feed;
   const newFeed = await this.getComponentFeed(url);
   this.store.dispatch(
     ac.BroadcastToContent({
       type: at.DISCOVERY_STREAM_FEED_UPDATE,
       data: {
         feed: newFeed,
         url,
       },
     })
   );
 }

 getExperimentInfo() {
   // We want to know if the user is in an experiment or rollout,
   // but we prioritize experiments over rollouts.
   const experimentMetadata =
     lazy.NimbusFeatures.pocketNewtab.getEnrollmentMetadata();

   let experimentName = experimentMetadata?.slug ?? "";
   let experimentBranch = experimentMetadata?.branch ?? "";

   return {
     experimentName,
     experimentBranch,
   };
 }

 // eslint-disable-next-line max-statements
 async getComponentFeed(feedUrl, isStartup) {
   const cachedData = (await this.cache.get()) || {};
   const prefs = this.store.getState().Prefs.values;
   const sectionsEnabled = prefs[PREF_SECTIONS_ENABLED];
   // Should we fetch /curated-recommendations over OHTTP
   const merinoOhttpEnabled = prefs[PREF_MERINO_OHTTP];
   let sections = [];
   const { feeds } = cachedData;

   let feed = feeds ? feeds[feedUrl] : null;
   if (this.isExpired({ cachedData, key: "feed", url: feedUrl, isStartup })) {
     const options = this.formatComponentFeedRequest(
       cachedData.sectionPersonalization
     );

     const feedResponse = await this.fetchFromEndpoint(
       feedUrl,
       options,
       merinoOhttpEnabled
     );

     if (feedResponse) {
       const { settings = {} } = feedResponse;
       let { recommendations } = feedResponse;

       recommendations = feedResponse.data.map(item => ({
         id: item.corpusItemId || item.scheduledCorpusItemId || item.tileId,
         scheduled_corpus_item_id: item.scheduledCorpusItemId,
         corpus_item_id: item.corpusItemId,
         features: item.features,
         excerpt: item.excerpt,
         icon_src: item.iconUrl,
         isTimeSensitive: item.isTimeSensitive,
         publisher: item.publisher,
         raw_image_src: item.imageUrl,
         received_rank: item.receivedRank,
         recommended_at: feedResponse.recommendedAt,
         title: item.title,
         topic: item.topic,
         url: item.url,
       }));

       if (sectionsEnabled) {
         const useClientLayout =
           this.store.getState().Prefs.values[PREF_CLIENT_LAYOUT_ENABLED];

         for (const [sectionKey, sectionData] of Object.entries(
           feedResponse.feeds
         )) {
           if (sectionData) {
             for (const item of sectionData.recommendations) {
               recommendations.push({
                 id:
                   item.corpusItemId ||
                   item.scheduledCorpusItemId ||
                   item.tileId,
                 scheduled_corpus_item_id: item.scheduledCorpusItemId,
                 corpus_item_id: item.corpusItemId,
                 url: item.url,
                 title: item.title,
                 topic: item.topic,
                 features: item.features,
                 excerpt: item.excerpt,
                 publisher: item.publisher,
                 raw_image_src: item.imageUrl,
                 received_rank: item.receivedRank,
                 server_score: item.serverScore,
                 recommended_at: feedResponse.recommendedAt,
                 section: sectionKey,
                 icon_src: item.iconUrl,
                 isTimeSensitive: item.isTimeSensitive,
               });
             }

             sections.push({
               sectionKey,
               title: sectionData.title,
               subtitle: sectionData.subtitle || "",
               receivedRank: sectionData.receivedFeedRank,
               layout: sectionData.layout,
               iab: sectionData.iab,
               // property if initially shown (with interest picker)
               visible: sectionData.isInitiallyVisible,
             });
           }
         }

         if (useClientLayout || sections.some(s => !s.layout)) {
           sections.sort((a, b) => a.receivedRank - b.receivedRank);

           sections.forEach((section, index) => {
             if (useClientLayout || !section.layout) {
               section.layout =
                 lazy.DEFAULT_SECTION_LAYOUT[
                   index % lazy.DEFAULT_SECTION_LAYOUT.length
                 ];
             }
           });
         }
       }

       const { data: scoredItems, personalized } = await this.scoreItems(
         recommendations,
         "feed"
       );

       if (sections.length) {
         const visibleSections = sections
           .filter(({ visible }) => visible)
           .sort((a, b) => a.receivedRank - b.receivedRank)
           .map(section => section.sectionKey)
           .join(",");

         // after the request only show the sections that are
         // initially visible and only keep the initial order (determined by the server)
         this.store.dispatch(
           ac.SetPref(PREF_VISIBLE_SECTIONS, visibleSections)
         );
       }

       // This assigns the section title to the interestPicker.sections
       // object to more easily access the title in JSX files
       if (
         feedResponse.interestPicker &&
         feedResponse.interestPicker.sections
       ) {
         feedResponse.interestPicker.sections =
           feedResponse.interestPicker.sections.map(section => {
             const { sectionId } = section;
             const title = sections.find(
               ({ sectionKey }) => sectionKey === sectionId
             )?.title;
             return { sectionId, title };
           });
       }
       if (feedResponse.inferredLocalModel) {
         this.store.dispatch(
           ac.AlsoToMain({
             type: at.INFERRED_PERSONALIZATION_MODEL_UPDATE,
             data: feedResponse.inferredLocalModel || {},
           })
         );
       }
       // We can cleanup any impressions we have that are old before we rotate.
       // In theory we can do this anywhere, but doing it just before rotate is optimal.
       // Rotate is also the only place that uses these impressions.
       await this.cleanUpTopRecImpressions();
       const rotatedItems = await this.rotate(scoredItems);

       const { data: filteredResults } =
         await this.filterBlocked(rotatedItems);
       this.componentFeedFetched = true;
       feed = {
         lastUpdated: Date.now(),
         personalized,
         sectionsEnabled,
         data: {
           settings,
           sections,
           interestPicker: feedResponse.interestPicker || {},
           recommendations: filteredResults,
           surfaceId: feedResponse.surfaceId || "",
           status: "success",
         },
       };
     } else {
       console.error("No response for feed");
     }
   }

   // if surfaceID is availible either through the cache or the response set value in Glean
   if (prefs[PREF_PRIVATE_PING_ENABLED] && feed.data.surfaceId) {
     Glean.newtabContent.surfaceId.set(feed.data.surfaceId);
     this.store.dispatch(ac.SetPref(PREF_SURFACE_ID, feed.data.surfaceId));
   }

   // If we have no feed at this point, both fetch and cache failed for some reason.
   return (
     feed || {
       data: {
         status: "failed",
       },
     }
   );
 }

 formatComponentFeedRequest(sectionPersonalization = {}) {
   const prefs = this.store.getState().Prefs.values;
   const inferredPersonalization =
     prefs[PREF_USER_INFERRED_PERSONALIZATION] &&
     prefs[PREF_SYSTEM_INFERRED_PERSONALIZATION];
   const merinoOhttpEnabled = prefs[PREF_MERINO_OHTTP];
   const headers = new Headers();
   const topicSelectionEnabled = prefs[PREF_TOPIC_SELECTION_ENABLED];
   const topicsString = prefs[PREF_SELECTED_TOPICS];
   const topics = topicSelectionEnabled
     ? topicsString
         .split(",")
         .map(s => s.trim())
         .filter(item => item)
     : [];

   // Should we pass the experiment branch and slug to the Merino feed request.
   const prefMerinoFeedExperiment = Services.prefs.getBoolPref(
     PREF_MERINO_FEED_EXPERIMENT
   );

   // convert section to array to match what merino is expecting
   const sections = Object.entries(sectionPersonalization).map(
     ([sectionId, data]) => ({
       sectionId,
       isFollowed: data.isFollowed,
       isBlocked: data.isBlocked,
       ...(data.followedAt && { followedAt: data.followedAt }),
     })
   );

   // To display the inline interest picker pass `enableInterestPicker` into the request
   const interestPickerEnabled = prefs[PREF_INTEREST_PICKER_ENABLED];

   let inferredInterests = null;
   if (inferredPersonalization && merinoOhttpEnabled) {
     inferredInterests =
       this.store.getState().InferredPersonalization
         ?.coarsePrivateInferredInterests || {};
     if (prefs[PREF_INFERRED_INTERESTS_OVERRIDE]) {
       try {
         inferredInterests = JSON.parse(
           prefs[PREF_INFERRED_INTERESTS_OVERRIDE]
         );
       } catch (ex) {
         console.error("Invalid format json for inferred interest override.");
       }
     }
   }

   const requestMetadata = {
     utc_offset: prefs.inferredPersonalizationConfig
       ?.normalized_time_zone_offset
       ? lazy.NewTabUtils.getUtcOffset(prefs[PREF_SURFACE_ID])
       : undefined,
     inferredInterests,
   };
   headers.append("content-type", "application/json");
   let body = {
     ...(prefMerinoFeedExperiment ? this.getExperimentInfo() : {}),
     ...requestMetadata,
     locale: this.locale,
     region: this.region,
     topics,
     sections,
     enableInterestPicker: !!interestPickerEnabled,
   };

   const sectionsEnabled = prefs[PREF_SECTIONS_ENABLED];

   if (sectionsEnabled) {
     body.feeds = ["sections"];
   }

   return {
     method: "POST",
     headers,
     body: JSON.stringify(body),
   };
 }

 /**
  * Called at startup to update cached data in the background.
  */
 async _maybeUpdateCachedData() {
   const expirationPerComponent = await this._checkExpirationPerComponent();
   // Pass in `store.dispatch` to send the updates only to main
   if (expirationPerComponent.spocs) {
     await this.loadSpocs(this.store.dispatch);
   }
   if (expirationPerComponent.feeds) {
     await this.loadComponentFeeds(this.store.dispatch);
   }
 }

 async scoreFeeds(feedsState) {
   if (feedsState.data) {
     const feeds = {};
     const feedsPromises = Object.keys(feedsState.data).map(url => {
       let feed = feedsState.data[url];
       if (feed.personalized) {
         // Feed was previously personalized then cached, we don't need to do this again.
         return Promise.resolve();
       }
       const feedPromise = this.scoreItems(feed.data.recommendations, "feed");
       feedPromise.then(({ data: scoredItems, personalized }) => {
         feed = {
           ...feed,
           personalized,
           data: {
             ...feed.data,
             recommendations: scoredItems,
           },
         };

         feeds[url] = feed;

         this.store.dispatch(
           ac.AlsoToPreloaded({
             type: at.DISCOVERY_STREAM_FEED_UPDATE,
             data: {
               feed,
               url,
             },
           })
         );
       });
       return feedPromise;
     });
     await Promise.all(feedsPromises);
     await this.cache.set("feeds", feeds);
   }
 }

 async scoreSpocs(spocsState) {
   const spocsResultPromises = this.getPlacements().map(async placement => {
     const nextSpocs = spocsState.data[placement.name] || {};
     const { items } = nextSpocs;

     if (nextSpocs.personalized || !items || !items.length) {
       return;
     }

     const { data: scoreResult, personalized } = await this.scoreItems(
       items,
       "spocs"
     );

     spocsState.data = {
       ...spocsState.data,
       [placement.name]: {
         ...nextSpocs,
         personalized,
         items: scoreResult,
       },
     };
   });
   await Promise.all(spocsResultPromises);

   // Update cache here so we don't need to re calculate scores on loads from cache.
   // Related Bug 1606276
   await this.cache.set("spocs", {
     lastUpdated: spocsState.lastUpdated,
     spocs: spocsState.data,
     spocsOnDemand: this.spocsOnDemand,
     spocsCacheUpdateTime: this.spocsCacheUpdateTime,
   });
   this.store.dispatch(
     ac.AlsoToPreloaded({
       type: at.DISCOVERY_STREAM_SPOCS_UPDATE,
       data: {
         lastUpdated: spocsState.lastUpdated,
         spocs: spocsState.data,
         spocsOnDemand: this.spocsOnDemand,
         spocsCacheUpdateTime: this.spocsCacheUpdateTime,
       },
     })
   );
 }

 /**
  * @typedef {object} RefreshAll
  * @property {boolean} updateOpenTabs - Sends updates to open tabs immediately if true,
  *                                      updates in background if false
  * @property {boolean} isStartup - When the function is called at browser startup
  *
  * Refreshes component feeds, and spocs in order if caches have expired.
  * @param {RefreshAll} options
  */
 async refreshAll(options = {}) {
   const { updateOpenTabs, isStartup, isSystemTick } = options;

   const dispatch = updateOpenTabs
     ? action => this.store.dispatch(ac.BroadcastToContent(action))
     : this.store.dispatch;

   this.loadLayout(dispatch, isStartup);
   if (this.showStories) {
     const spocsStartupCacheEnabled =
       this.store.getState().Prefs.values[PREF_SPOCS_STARTUP_CACHE_ENABLED];
     const promises = [];

     // We don't want to make spoc requests during system tick if on demand is on.
     if (!(this.spocsOnDemand && isSystemTick)) {
       const spocsPromise = this.loadSpocs(
         dispatch,
         isStartup && spocsStartupCacheEnabled
       ).catch(error =>
         console.error("Error trying to load spocs feeds:", error)
       );
       promises.push(spocsPromise);
     }
     const storiesPromise = this.loadComponentFeeds(dispatch, isStartup).catch(
       error => console.error("Error trying to load component feeds:", error)
     );
     promises.push(storiesPromise);
     await Promise.all(promises);
     // We don't need to check onDemand here,
     // even though _maybeUpdateCachedData fetches spocs.
     // This is because isStartup and isSystemTick can never both be true.
     if (isStartup) {
       // We don't pass isStartup in _maybeUpdateCachedData on purpose,
       // because startup loads have a longer cache timer,
       // and we want this to update in the background sooner.
       await this._maybeUpdateCachedData();
     }
   }
 }

 // We have to rotate stories on the client so that
 // active stories are at the front of the list, followed by stories that have expired
 // impressions i.e. have been displayed for longer than DEFAULT_RECS_ROTATION_TIME.
 async rotate(recommendations) {
   const cachedData = (await this.cache.get()) || {};
   const impressions = cachedData.recsImpressions;

   // If we have no impressions, don't bother checking.
   if (!impressions) {
     return recommendations;
   }

   const expired = [];
   const active = [];
   for (const item of recommendations) {
     if (
       impressions[item.id] &&
       Date.now() - impressions[item.id] >= DEFAULT_RECS_ROTATION_TIME
     ) {
       expired.push(item);
     } else {
       active.push(item);
     }
   }
   return active.concat(expired);
 }

 enableStories() {
   if (this.config.enabled) {
     // If stories are being re enabled, ensure we have stories.
     this.refreshAll({ updateOpenTabs: true });
   }
 }

 async enable(options = {}) {
   await this.refreshAll(options);
   this.loaded = true;
 }

 async reset() {
   this.resetDataPrefs();
   await this.resetCache();
   this.resetState();
 }

 async resetCache() {
   await this.resetAllCache();
 }

 async resetContentCache() {
   await this.cache.set("feeds", {});
   await this.cache.set("spocs", {});
   await this.cache.set("recsImpressions", {});
 }

 async resetBlocks() {
   await this.cache.set("recsBlocks", {});
   const cachedData = (await this.cache.get()) || {};
   let blocks = cachedData.recsBlocks || {};

   this.store.dispatch({
     type: at.DISCOVERY_STREAM_DEV_BLOCKS,
     data: blocks,
   });
   // Update newtab after clearing blocks.
   await this.refreshAll({ updateOpenTabs: true });
 }

 async resetContentFeed() {
   await this.cache.set("feeds", {});
 }

 async resetSpocs() {
   await this.cache.set("spocs", {});
 }

 async resetAllCache() {
   await this.resetContentCache();
   // Reset in-memory caches.
   this._isContextualAds = undefined;
   this._doLocalInferredRerank = undefined;
   this._spocsCacheUpdateTime = undefined;
   this._spocsOnDemand = undefined;
 }

 resetDataPrefs() {
   this.writeDataPref(PREF_SPOC_IMPRESSIONS, {});
   this.writeDataPref(PREF_FLIGHT_BLOCKS, {});
 }

 resetState() {
   // Reset reducer
   this.store.dispatch(
     ac.BroadcastToContent({ type: at.DISCOVERY_STREAM_LAYOUT_RESET })
   );
   this.setupPrefs(false /* isStartup */);
   this.loaded = false;
 }

 async onPrefChange() {
   // We always want to clear the cache/state if the pref has changed
   await this.reset();
   if (this.config.enabled) {
     // Load data from all endpoints
     await this.enable({ updateOpenTabs: true });
   }
 }

 // This is a request to change the config from somewhere.
 // Can be from a specific pref related to Discovery Stream,
 // or can be a generic request from an external feed that
 // something changed.
 configReset() {
   this._prefCache.config = null;
   this.store.dispatch(
     ac.BroadcastToContent({
       type: at.DISCOVERY_STREAM_CONFIG_CHANGE,
       data: this.config,
     })
   );
 }

 recordFlightImpression(flightId) {
   let impressions = this.readDataPref(PREF_SPOC_IMPRESSIONS);

   const timeStamps = impressions[flightId] || [];
   timeStamps.push(Date.now());
   impressions = { ...impressions, [flightId]: timeStamps };

   this.writeDataPref(PREF_SPOC_IMPRESSIONS, impressions);
 }

 async recordTopRecImpression(recId) {
   const cachedData = (await this.cache.get()) || {};
   let impressions = cachedData.recsImpressions || {};

   if (!impressions[recId]) {
     impressions = { ...impressions, [recId]: Date.now() };
     await this.cache.set("recsImpressions", impressions);

     this.store.dispatch({
       type: at.DISCOVERY_STREAM_DEV_IMPRESSIONS,
       data: impressions,
     });
   }
 }

 async recordBlockRecId(recId) {
   const cachedData = (await this.cache.get()) || {};
   let blocks = cachedData.recsBlocks || {};

   if (!blocks[recId]) {
     blocks[recId] = 1;
     await this.cache.set("recsBlocks", blocks);

     this.store.dispatch({
       type: at.DISCOVERY_STREAM_DEV_BLOCKS,
       data: blocks,
     });
   }
 }

 recordBlockFlightId(flightId) {
   const unifiedAdsEnabled =
     this.store.getState().Prefs.values[PREF_UNIFIED_ADS_SPOCS_ENABLED];

   const flights = this.readDataPref(PREF_FLIGHT_BLOCKS);
   if (!flights[flightId]) {
     flights[flightId] = 1;
     this.writeDataPref(PREF_FLIGHT_BLOCKS, flights);

     if (unifiedAdsEnabled) {
       let blockList =
         this.store.getState().Prefs.values[PREF_UNIFIED_ADS_BLOCKED_LIST];

       let blockedAdsArray = [];

       // If prev ads have been blocked, convert CSV string to array
       if (blockList !== "") {
         blockedAdsArray = blockList
           .split(",")
           .map(s => s.trim())
           .filter(item => item);
       }

       blockedAdsArray.push(flightId);

       this.store.dispatch(
         ac.SetPref(PREF_UNIFIED_ADS_BLOCKED_LIST, blockedAdsArray.join(","))
       );
     }
   }
 }

 cleanUpFlightImpressionPref(data) {
   let flightIds = [];
   this.placementsForEach(placement => {
     const newSpocs = data[placement.name];
     if (!newSpocs) {
       return;
     }

     const items = newSpocs.items || [];
     flightIds = [...flightIds, ...items.map(s => `${s.flight_id}`)];
   });
   if (flightIds && flightIds.length) {
     this.cleanUpImpressionPref(
       id => !flightIds.includes(id),
       PREF_SPOC_IMPRESSIONS
     );
   }
 }

 // Clean up rec impressions that are old.
 async cleanUpTopRecImpressions() {
   await this.cleanUpImpressionCache(
     impression =>
       Date.now() - impression >= DEFAULT_RECS_IMPRESSION_EXPIRE_TIME,
     "recsImpressions"
   );
 }

 writeDataPref(pref, impressions) {
   this.store.dispatch(ac.SetPref(pref, JSON.stringify(impressions)));
 }

 readDataPref(pref) {
   const prefVal = this.store.getState().Prefs.values[pref];
   return prefVal ? JSON.parse(prefVal) : {};
 }

 async cleanUpImpressionCache(isExpired, cacheKey) {
   const cachedData = (await this.cache.get()) || {};
   let impressions = cachedData[cacheKey];
   let changed = false;

   if (impressions) {
     Object.keys(impressions).forEach(id => {
       if (isExpired(impressions[id])) {
         changed = true;
         delete impressions[id];
       }
     });

     if (changed) {
       await this.cache.set(cacheKey, impressions);

       this.store.dispatch({
         type: at.DISCOVERY_STREAM_DEV_IMPRESSIONS,
         data: impressions,
       });
     }
   }
 }

 cleanUpImpressionPref(isExpired, pref) {
   const impressions = this.readDataPref(pref);
   let changed = false;

   Object.keys(impressions).forEach(id => {
     if (isExpired(id)) {
       changed = true;
       delete impressions[id];
     }
   });

   if (changed) {
     this.writeDataPref(pref, impressions);
   }
 }

 async retreiveProfileAge() {
   let profileAccessor = await lazy.ProfileAge();
   let profileCreateTime = await profileAccessor.created;
   let timeNow = new Date().getTime();
   let profileAge = timeNow - profileCreateTime;
   // Convert milliseconds to days
   return profileAge / 1000 / 60 / 60 / 24;
 }

 topicSelectionImpressionEvent() {
   let counter =
     this.store.getState().Prefs.values[TOPIC_SELECTION_DISPLAY_COUNT];

   const newCount = counter + 1;
   this.store.dispatch(ac.SetPref(TOPIC_SELECTION_DISPLAY_COUNT, newCount));
   this.store.dispatch(
     ac.SetPref(TOPIC_SELECTION_LAST_DISPLAYED, `${new Date().getTime()}`)
   );
 }

 topicSelectionMaybeLaterEvent() {
   const age = this.retreiveProfileAge();
   const newProfile = age <= 1;
   const day = 24 * 60 * 60 * 1000;
   this.store.dispatch(
     ac.SetPref(
       TOPIC_SELECTION_DISPLAY_TIMEOUT,
       newProfile ? 3 * day : 7 * day
     )
   );
 }

 async onSpocsOnDemandUpdate() {
   if (this.spocsOnDemand) {
     const expirationPerComponent = await this._checkExpirationPerComponent();
     if (expirationPerComponent.spocs) {
       await this.loadSpocs(action =>
         this.store.dispatch(ac.BroadcastToContent(action))
       );
     }
   }
 }

 async onSystemTick() {
   // Only refresh when enabled and after initial load has completed.
   if (!this.config.enabled || !this.loaded) {
     return;
   }

   const expirationPerComponent = await this._checkExpirationPerComponent();
   let expired = false;

   if (this.spocsOnDemand) {
     // With on-demand only feeds can trigger a refresh.
     expired = expirationPerComponent.feeds;
   } else {
     // Without on-demand both feeds or spocs can trigger a refresh.
     expired = expirationPerComponent.feeds || expirationPerComponent.spocs;
   }

   if (expired) {
     // We use isSystemTick so refreshAll can know to check onDemand
     await this.refreshAll({ updateOpenTabs: false, isSystemTick: true });
   }
 }

 async onTrainhopConfigChanged() {
   this.resetSpocsOnDemand();
 }

 async onPrefChangedAction(action) {
   switch (action.data.name) {
     case PREF_CONFIG:
     case PREF_ENABLED:
     case PREF_HARDCODED_BASIC_LAYOUT:
     case PREF_SPOCS_ENDPOINT:
     case PREF_SPOCS_ENDPOINT_QUERY:
     case PREF_SPOCS_CLEAR_ENDPOINT:
     case PREF_ENDPOINTS:
     case PREF_SPOC_POSITIONS:
     case PREF_UNIFIED_ADS_SPOCS_ENABLED:
     case PREF_SECTIONS_ENABLED:
     case PREF_INTEREST_PICKER_ENABLED:
       // This is a config reset directly related to Discovery Stream pref.
       this.configReset();
       break;
     case PREF_USER_INFERRED_PERSONALIZATION:
       this.configReset();
       this._isContextualAds = undefined;
       this._doLocalInferredRerank = undefined;
       await this.resetContentCache();
       break;
     case PREF_CONTEXTUAL_ADS:
     case PREF_SYSTEM_INFERRED_PERSONALIZATION:
       this._isContextualAds = undefined;
       this._doLocalInferredRerank = undefined;
       break;
     case PREF_SELECTED_TOPICS:
       this.store.dispatch(
         ac.BroadcastToContent({ type: at.DISCOVERY_STREAM_LAYOUT_RESET })
       );
       // Ensure at least a little bit of loading is seen, if this is too fast,
       // it's not clear to the user what just happened.
       this.store.dispatch(
         ac.BroadcastToContent({
           type: at.DISCOVERY_STREAM_TOPICS_LOADING,
           data: true,
         })
       );
       setTimeout(() => {
         this.store.dispatch(
           ac.BroadcastToContent({
             type: at.DISCOVERY_STREAM_TOPICS_LOADING,
             data: false,
           })
         );
       }, TOPIC_LOADING_TIMEOUT);
       this.loadLayout(
         a => this.store.dispatch(ac.BroadcastToContent(a)),
         false
       );

       // when topics have been updated, make a new request from merino and clear impression cap
       await this.cache.set("recsImpressions", {});
       await this.resetContentFeed();
       this.refreshAll({ updateOpenTabs: true });
       break;
     case PREF_USER_TOPSTORIES:
     case PREF_SYSTEM_TOPSTORIES:
       if (!this.showStories) {
         // Ensure we delete any remote data potentially related to spocs.
         this.clearSpocs();
       }
       if (action.data.value) {
         this.enableStories();
       }
       break;
     // Remove spocs if turned off.
     case PREF_SHOW_SPONSORED: {
       await this.updateOrRemoveSpocs();
       break;
     }
     case PREF_SPOCS_CACHE_ONDEMAND:
     case PREF_SPOCS_CACHE_TIMEOUT: {
       this.resetSpocsOnDemand();
       break;
     }
   }

   if (action.data.name === "pocketConfig") {
     await this.onPrefChange();
     this.setupPrefs(false /* isStartup */);
   }
   if (action.data.name === "trainhopConfig") {
     await this.onTrainhopConfigChanged(action);
   }
 }

 resetSpocsOnDemand() {
   // This is all we have to do, because we're just changing how often caches update.
   // No need to reset what is already fetched, we just care about the next check.
   this._spocsCacheUpdateTime = undefined;
   this._spocsOnDemand = undefined;
   this.store.dispatch({
     type: at.DISCOVERY_STREAM_SPOCS_ONDEMAND_RESET,
     data: {
       spocsOnDemand: this.spocsOnDemand,
       spocsCacheUpdateTime: this.spocsCacheUpdateTime,
     },
   });
 }

 async onAction(action) {
   switch (action.type) {
     case at.INIT:
       // During the initialization of Firefox:
       // 1. Set-up listeners and initialize the redux state for config;
       this.setupConfig(true /* isStartup */);
       this.setupPrefs(true /* isStartup */);
       // 2. If config.enabled is true, start loading data.
       if (this.config.enabled) {
         await this.enable({ updateOpenTabs: true, isStartup: true });
       }
       // This function is async but just for devtools,
       // so we don't need to wait for it.
       this.setupDevtoolsState(true /* isStartup */);
       break;
     case at.TOPIC_SELECTION_MAYBE_LATER:
       this.topicSelectionMaybeLaterEvent();
       break;
     case at.DISCOVERY_STREAM_DEV_BLOCKS_RESET:
       await this.resetBlocks();
       break;
     case at.DISCOVERY_STREAM_DEV_SYSTEM_TICK:
     case at.SYSTEM_TICK:
       await this.onSystemTick();
       break;
     case at.DISCOVERY_STREAM_SPOCS_ONDEMAND_UPDATE: {
       await this.onSpocsOnDemandUpdate();
       break;
     }
     case at.DISCOVERY_STREAM_DEV_SYNC_RS:
       lazy.RemoteSettings.pollChanges();
       break;
     case at.DISCOVERY_STREAM_DEV_EXPIRE_CACHE:
       // Personalization scores update at a slower interval than content, so in order to debug,
       // we want to be able to expire just content to trigger the earlier expire times.
       await this.resetContentCache();
       break;
     case at.DISCOVERY_STREAM_DEV_SHOW_PLACEHOLDER: {
       // We want to display the loading state permanently, for dev purposes.
       // We do this by resetting everything, loading the layout, and nothing else.
       // This essentially hangs because we never triggered the content load.
       await this.reset();
       this.loadLayout(
         a => this.store.dispatch(ac.BroadcastToContent(a)),
         false
       );
       break;
     }
     case at.DISCOVERY_STREAM_CONFIG_SET_VALUE:
       // Use the original string pref to then set a value instead of
       // this.config which has some modifications
       this.store.dispatch(
         ac.SetPref(
           PREF_CONFIG,
           JSON.stringify({
             ...JSON.parse(this.store.getState().Prefs.values[PREF_CONFIG]),
             [action.data.name]: action.data.value,
           })
         )
       );
       break;
     case at.DISCOVERY_STREAM_PERSONALIZATION_UPDATED:
       if (this.personalized) {
         const { feeds, spocs } = this.store.getState().DiscoveryStream;
         const spocsPersonalized =
           this.store.getState().Prefs.values?.pocketConfig?.spocsPersonalized;
         const recsPersonalized =
           this.store.getState().Prefs.values?.pocketConfig?.recsPersonalized;
         if (recsPersonalized && feeds.loaded) {
           this.scoreFeeds(feeds);
         }
         if (spocsPersonalized && spocs.loaded) {
           this.scoreSpocs(spocs);
         }
       }
       break;
     case at.DISCOVERY_STREAM_CONFIG_RESET:
       // This is a generic config reset likely related to an external feed pref.
       this.configReset();
       break;
     case at.DISCOVERY_STREAM_CONFIG_RESET_DEFAULTS:
       this.resetConfigDefauts();
       break;
     case at.DISCOVERY_STREAM_RETRY_FEED:
       this.retryFeed(action.data.feed);
       break;
     case at.DISCOVERY_STREAM_CONFIG_CHANGE:
       // When the config pref changes, load or unload data as needed.
       await this.onPrefChange();
       break;
     case at.DISCOVERY_STREAM_IMPRESSION_STATS:
       if (
         action.data.tiles &&
         action.data.tiles[0] &&
         action.data.tiles[0].id
       ) {
         this.recordTopRecImpression(action.data.tiles[0].id);
       }
       break;
     case at.DISCOVERY_STREAM_SPOC_IMPRESSION:
       if (this.showSponsoredStories) {
         this.recordFlightImpression(action.data.flightId);

         // Apply frequency capping to SPOCs in the redux store, only update the
         // store if the SPOCs are changed.
         const spocsState = this.store.getState().DiscoveryStream.spocs;

         let frequencyCapped = [];
         this.placementsForEach(placement => {
           const spocs = spocsState.data[placement.name];
           if (!spocs || !spocs.items) {
             return;
           }

           const { data: capResult, filtered } = this.frequencyCapSpocs(
             spocs.items
           );
           frequencyCapped = [...frequencyCapped, ...filtered];

           spocsState.data = {
             ...spocsState.data,
             [placement.name]: {
               ...spocs,
               items: capResult,
             },
           };
         });

         if (frequencyCapped.length) {
           // Update cache here so we don't need to re calculate frequency caps on loads from cache.
           await this.cache.set("spocs", {
             lastUpdated: spocsState.lastUpdated,
             spocs: spocsState.data,
             spocsOnDemand: this.spocsOnDemand,
             spocsCacheUpdateTime: this.spocsCacheUpdateTime,
           });

           this.store.dispatch(
             ac.AlsoToPreloaded({
               type: at.DISCOVERY_STREAM_SPOCS_UPDATE,
               data: {
                 lastUpdated: spocsState.lastUpdated,
                 spocs: spocsState.data,
                 spocsOnDemand: this.spocsOnDemand,
                 spocsCacheUpdateTime: this.spocsCacheUpdateTime,
               },
             })
           );
         }
       }
       break;

     // This is fired from the browser, it has no concept of spocs, flights or pocket.
     // We match the blocked url with our available story urls to see if there is a match.
     // I suspect we *could* instead do this in BLOCK_URL but I'm not sure.
     case at.PLACES_LINK_BLOCKED: {
       const feedsState = this.store.getState().DiscoveryStream.feeds;
       const feeds = {};

       for (const url of Object.keys(feedsState.data)) {
         let feed = feedsState.data[url];

         const { data: filteredResults } = await this.filterBlocked(
           feed.data.recommendations
         );

         feed = {
           ...feed,
           data: {
             ...feed.data,
             recommendations: filteredResults,
           },
         };

         feeds[url] = feed;
       }

       await this.cache.set("feeds", feeds);

       if (this.showSponsoredStories) {
         let blockedItems = [];
         const spocsState = this.store.getState().DiscoveryStream.spocs;

         this.placementsForEach(placement => {
           const spocs = spocsState.data[placement.name];
           if (spocs && spocs.items && spocs.items.length) {
             const blockedResults = [];
             const blocks = spocs.items.filter(s => {
               const blocked = s.url === action.data.url;
               if (!blocked) {
                 blockedResults.push(s);
               }
               return blocked;
             });

             blockedItems = [...blockedItems, ...blocks];

             spocsState.data = {
               ...spocsState.data,
               [placement.name]: {
                 ...spocs,
                 items: blockedResults,
               },
             };
           }
         });

         if (blockedItems.length) {
           // Update cache here so we don't need to re calculate blocks on loads from cache.
           await this.cache.set("spocs", {
             lastUpdated: spocsState.lastUpdated,
             spocs: spocsState.data,
             spocsOnDemand: this.spocsOnDemand,
             spocsCacheUpdateTime: this.spocsCacheUpdateTime,
           });

           // If we're blocking a spoc, we want open tabs to have
           // a slightly different treatment from future tabs.
           // AlsoToPreloaded updates the source data and preloaded tabs with a new spoc.
           // BroadcastToContent updates open tabs with a non spoc instead of a new spoc.
           this.store.dispatch(
             ac.AlsoToPreloaded({
               type: at.DISCOVERY_STREAM_LINK_BLOCKED,
               data: action.data,
             })
           );
           this.store.dispatch(
             ac.BroadcastToContent({
               type: at.DISCOVERY_STREAM_SPOC_BLOCKED,
               data: action.data,
             })
           );
           break;
         }
       }

       this.store.dispatch(
         ac.BroadcastToContent({
           type: at.DISCOVERY_STREAM_LINK_BLOCKED,
           data: action.data,
         })
       );
       break;
     }
     case at.UNINIT:
       // When this feed is shutting down:
       this.uninitPrefs();
       this._recommendationProvider = null;
       break;
     case at.BLOCK_URL: {
       // If we block a story that also has a flight_id
       // we want to record that as blocked too.
       // This is because a single flight might have slightly different urls.
       for (const site of action.data) {
         const { flight_id, tile_id } = site;
         if (flight_id) {
           this.recordBlockFlightId(flight_id);
         }
         if (tile_id) {
           await this.recordBlockRecId(tile_id);
         }
       }
       break;
     }
     case at.PREF_CHANGED:
       await this.onPrefChangedAction(action);
       break;
     case at.TOPIC_SELECTION_IMPRESSION:
       this.topicSelectionImpressionEvent();
       break;
     case at.SECTION_PERSONALIZATION_SET:
       await this.cache.set("sectionPersonalization", action.data);
       this.store.dispatch(
         ac.BroadcastToContent({
           type: at.SECTION_PERSONALIZATION_UPDATE,
           data: action.data,
         })
       );
       break;
     case at.INFERRED_PERSONALIZATION_MODEL_UPDATE:
       await this.cache.set("inferredModel", action.data);
       break;
     case at.ADS_UPDATE_SPOCS:
       await this.updateOrRemoveSpocs();
       break;
   }
 }
}

/* This function generates a hardcoded layout each call.
  This is because modifying the original object would
  persist across pref changes and system_tick updates.

  NOTE: There is some branching logic in the template.
    `spocsUrl` Changing the url for spocs is used for adding a siteId query param.
    `feedUrl` Where to fetch stories from.
    `items` How many items to include in the primary card grid.
    `spocPositions` Changes the position of spoc cards.
    `spocPlacementData` Used to set the spoc content.
    `hybridLayout` Changes cards to smaller more compact cards only for specific breakpoints.
    `hideCardBackground` Removes Pocket card background and borders.
    `fourCardLayout` Enable four Pocket cards per row.
    `newFooterSection` Changes the layout of the topics section.
    `compactGrid` Reduce the number of pixels between the Pocket cards.
    `ctaButtonSponsors` An array of sponsors we want to show a cta button on the card for.
    `ctaButtonVariant` Sets the variant for the cta sponsor button.
*/
getHardcodedLayout = ({
 spocsUrl = SPOCS_URL,
 feedUrl,
 items = 21,
 spocPositions = [1, 5, 7, 11, 18, 20],
 spocPlacementData = { ad_types: [3617], zone_ids: [217758, 217995] },
 widgetPositions = [],
 widgetData = [],
 hybridLayout = false,
 hideCardBackground = false,
 fourCardLayout = false,
 newFooterSection = false,
 compactGrid = false,
 ctaButtonSponsors = [],
 ctaButtonVariant = "",
 pocketStoriesHeadlineId = "newtab-section-header-stories",
}) => ({
 lastUpdate: Date.now(),
 spocs: {
   url: spocsUrl,
 },
 layout: [
   {
     width: 12,
     components: [
       {
         type: "TopSites",
         header: {
           title: {
             id: "newtab-section-header-topsites",
           },
         },
         properties: {},
       },
       {
         type: "Message",
         header: {
           title: {
             id: pocketStoriesHeadlineId,
           },
           subtitle: "",
           link_text: {
             id: "newtab-pocket-learn-more",
           },
           link_url: "",
           icon: "chrome://global/skin/icons/pocket.svg",
         },
         styles: {
           ".ds-message": "margin-block-end: -20px",
         },
       },
       {
         type: "CardGrid",
         properties: {
           items,
           hybridLayout,
           hideCardBackground,
           fourCardLayout,
           compactGrid,
           ctaButtonSponsors,
           ctaButtonVariant,
         },
         widgets: {
           positions: widgetPositions.map(position => {
             return { index: position };
           }),
           data: widgetData,
         },
         cta_variant: "link",
         header: {
           title: "",
         },
         placement: {
           name: "newtab_spocs",
           ad_types: spocPlacementData.ad_types,
           zone_ids: spocPlacementData.zone_ids,
         },
         feed: {
           embed_reference: null,
           url: feedUrl,
         },
         spocs: {
           probability: 1,
           positions: spocPositions.map(position => {
             return { index: position };
           }),
         },
       },
       {
         type: "Navigation",
         newFooterSection,
         properties: {
           alignment: "left-align",
           extraLinks: [
             {
               name: "Career",
               url: "https://getpocket.com/explore/career?utm_source=pocket-newtab",
             },
             {
               name: "Technology",
               url: "https://getpocket.com/explore/technology?utm_source=pocket-newtab",
             },
           ],
           privacyNoticeURL: {
             url: "https://www.mozilla.org/privacy/firefox/#recommend-relevant-content",
             title: {
               id: "newtab-section-menu-privacy-notice",
             },
           },
         },
         styles: {
           ".ds-navigation": "margin-block-start: -10px;",
         },
       },
       ...(newFooterSection
         ? [
             {
               type: "PrivacyLink",
               properties: {
                 url: "https://www.mozilla.org/privacy/firefox/",
                 title: {
                   id: "newtab-section-menu-privacy-notice",
                 },
               },
             },
           ]
         : []),
     ],
   },
 ],
});