tor-browser

TopSites.sys.mjs (43868B)

---

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
// eslint-disable-next-line mozilla/use-static-import
const { AppConstants } = ChromeUtils.importESModule(
 "resource://gre/modules/AppConstants.sys.mjs"
);

import {
 getDomain,
 TippyTopProvider,
} from "resource:///modules/topsites/TippyTopProvider.sys.mjs";
import { Dedupe } from "resource:///modules/Dedupe.sys.mjs";
import { TOP_SITES_MAX_SITES_PER_ROW } from "resource:///modules/topsites/constants.mjs";
import {
 CUSTOM_SEARCH_SHORTCUTS,
 checkHasSearchEngine,
 getSearchProvider,
} from "moz-src:///toolkit/components/search/SearchShortcuts.sys.mjs";

const lazy = {};

ChromeUtils.defineESModuleGetters(lazy, {
 FilterAdult: "resource:///modules/FilterAdult.sys.mjs",
 LinksCache: "resource:///modules/LinksCache.sys.mjs",
 NetUtil: "resource://gre/modules/NetUtil.sys.mjs",
 NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
 PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
 Region: "resource://gre/modules/Region.sys.mjs",
 RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
});

ChromeUtils.defineLazyGetter(lazy, "log", () => {
 const { Logger } = ChromeUtils.importESModule(
   "resource://messaging-system/lib/Logger.sys.mjs"
 );
 return new Logger("TopSites");
});

ChromeUtils.defineLazyGetter(lazy, "pageFrecencyThreshold", () => {
 // @backward-compat { version 147 }
 // Frecency was changed in 147 Nightly. This is a pre-cautionary measure
 // for train-hopping.
 if (Services.vc.compare(AppConstants.MOZ_APP_VERSION, "147.0a1") >= 0) {
   // 30 days ago, 5 visits. The threshold avoids one non-typed visit from
   // immediately being included in recent history to mimic the original
   // threshold which aimed to prevent first-run visits from being included in
   // Top Sites.
   return lazy.PlacesUtils.history.pageFrecencyThreshold(30, 5, false);
 }
 // The old threshold used for classic frecency: Slightly over one visit.
 return 101;
});

export const DEFAULT_TOP_SITES = [];

const MIN_FAVICON_SIZE = 96;
const PINNED_FAVICON_PROPS_TO_MIGRATE = [
 "favicon",
 "faviconRef",
 "faviconSize",
];

// Preferences
const NO_DEFAULT_SEARCH_TILE_PREF =
 "browser.newtabpage.activity-stream.improvesearch.noDefaultSearchTile";
const SEARCH_SHORTCUTS_HAVE_PINNED_PREF =
 "browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts.havePinned";
// TODO: Rename this when re-subscribing to the search engines pref.
const SEARCH_SHORTCUTS_ENGINES =
 "browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts.searchEngines";
const TOP_SITE_SEARCH_SHORTCUTS_PREF =
 "browser.newtabpage.activity-stream.improvesearch.topSiteSearchShortcuts";
const TOP_SITES_ROWS_PREF = "browser.newtabpage.activity-stream.topSitesRows";

// Search experiment stuff
const SEARCH_FILTERS = [
 "google",
 "search.yahoo",
 "yahoo",
 "bing",
 "ask",
 "duckduckgo",
];

const REMOTE_SETTING_DEFAULTS_PREF = "browser.topsites.useRemoteSetting";
const DEFAULT_SITES_OVERRIDE_PREF =
 "browser.newtabpage.activity-stream.default.sites";
const DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH = "browser.topsites.experiment.";

function getShortHostnameForCurrentSearch() {
 const url = lazy.NewTabUtils.shortHostname(
   Services.search.defaultEngine.searchUrlDomain
 );
 return url;
}

class _TopSites {
 #hasObservers = false;
 /**
  * A Promise used to determine if initialization is complete.
  *
  * @type {Promise}
  */
 #initPromise = null;
 #searchShortcuts = [];
 #sites = [];

 constructor() {
   this._tippyTopProvider = new TippyTopProvider();
   ChromeUtils.defineLazyGetter(
     this,
     "_currentSearchHostname",
     getShortHostnameForCurrentSearch
   );
   this.dedupe = new Dedupe(this._dedupeKey);
   this.frecentCache = new lazy.LinksCache(
     lazy.NewTabUtils.activityStreamLinks,
     "getTopSites",
     [],
     (oldOptions, newOptions) =>
       // Refresh if no old options or requesting more items
       !(oldOptions.numItems >= newOptions.numItems)
   );
   this.pinnedCache = new lazy.LinksCache(
     lazy.NewTabUtils.pinnedLinks,
     "links",
     [...PINNED_FAVICON_PROPS_TO_MIGRATE]
   );
   this._faviconProvider = new FaviconProvider();
   this.handlePlacesEvents = this.handlePlacesEvents.bind(this);
 }

 /**
  * Initializes the TopSites module.
  *
  * @returns {Promise}
  */
 async init() {
   if (this.#initPromise) {
     return this.#initPromise;
   }
   this.#initPromise = (async () => {
     lazy.log.debug("Initializing TopSites.");
     this.#addObservers();
     await this._readDefaults({ isStartup: true });
     // TopSites was initialized by the store calling the initialization
     // function and then updating custom search shortcuts. Since
     // initialization now happens upon the first retrieval of sites, we move
     // the update custom search shortcuts here.
     await this.updateCustomSearchShortcuts(true);
   })();
   return this.#initPromise;
 }

 uninit() {
   lazy.log.debug("Un-initializing TopSites.");
   this.#removeObservers();
   this.#searchShortcuts = [];
   this.#sites = [];
   this.#initPromise = null;
   this.frecentCache.expire();
   this.pinnedCache.expire();
 }

 #addObservers() {
   if (this.#hasObservers) {
     return;
   }
   // If the feed was previously disabled PREFS_INITIAL_VALUES was never received
   Services.obs.addObserver(this, "browser-search-engine-modified");
   Services.obs.addObserver(this, "browser-region-updated");
   Services.obs.addObserver(this, "newtab-linkBlocked");
   Services.prefs.addObserver(REMOTE_SETTING_DEFAULTS_PREF, this);
   Services.prefs.addObserver(DEFAULT_SITES_OVERRIDE_PREF, this);
   Services.prefs.addObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this);
   Services.prefs.addObserver(NO_DEFAULT_SEARCH_TILE_PREF, this);
   Services.prefs.addObserver(SEARCH_SHORTCUTS_ENGINES, this);
   Services.prefs.addObserver(TOP_SITES_ROWS_PREF, this);
   Services.prefs.addObserver(TOP_SITE_SEARCH_SHORTCUTS_PREF, this);
   lazy.PlacesUtils.observers.addListener(
     ["bookmark-added", "bookmark-removed", "history-cleared", "page-removed"],
     this.handlePlacesEvents
   );
   this.#hasObservers = true;
 }

 #removeObservers() {
   if (!this.#hasObservers) {
     return;
   }
   Services.obs.removeObserver(this, "browser-search-engine-modified");
   Services.obs.removeObserver(this, "browser-region-updated");
   Services.obs.removeObserver(this, "newtab-linkBlocked");
   Services.prefs.removeObserver(REMOTE_SETTING_DEFAULTS_PREF, this);
   Services.prefs.removeObserver(DEFAULT_SITES_OVERRIDE_PREF, this);
   Services.prefs.removeObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this);
   Services.prefs.removeObserver(NO_DEFAULT_SEARCH_TILE_PREF, this);
   Services.prefs.removeObserver(SEARCH_SHORTCUTS_ENGINES, this);
   Services.prefs.removeObserver(TOP_SITES_ROWS_PREF, this);
   Services.prefs.removeObserver(TOP_SITE_SEARCH_SHORTCUTS_PREF, this);
   lazy.PlacesUtils.observers.removeListener(
     ["bookmark-added", "bookmark-removed", "history-cleared", "page-removed"],
     this.handlePlacesEvents
   );
   this.#hasObservers = false;
 }

 _reset() {
   // Allow automated tests to reset the internal state of the component.
   if (Cu.isInAutomation) {
     this.#searchShortcuts = [];
     this.#sites = [];
   }
 }

 observe(subj, topic, data) {
   switch (topic) {
     case "browser-search-engine-modified":
       // We should update the current top sites if the search engine has been changed since
       // the search engine that gets filtered out of top sites has changed.
       // We also need to drop search shortcuts when their engine gets removed / hidden.
       if (
         data === "engine-default" &&
         Services.prefs.getBoolPref(NO_DEFAULT_SEARCH_TILE_PREF, true)
       ) {
         delete this._currentSearchHostname;
         this._currentSearchHostname = getShortHostnameForCurrentSearch();
       }
       this.refresh({ broadcast: true });
       break;
     case "browser-region-updated":
       this._readDefaults();
       break;
     case "newtab-linkBlocked":
       this.frecentCache.expire();
       this.pinnedCache.expire();
       this.refresh();
       break;
     case "nsPref:changed":
       switch (data) {
         case DEFAULT_SITES_OVERRIDE_PREF:
         case REMOTE_SETTING_DEFAULTS_PREF:
           this._readDefaults();
           break;
         case NO_DEFAULT_SEARCH_TILE_PREF:
           this.refresh();
           break;
         case TOP_SITES_ROWS_PREF:
         case SEARCH_SHORTCUTS_ENGINES:
           this.refresh();
           break;
         case TOP_SITE_SEARCH_SHORTCUTS_PREF:
           if (Services.prefs.getBoolPref(TOP_SITE_SEARCH_SHORTCUTS_PREF)) {
             this.updateCustomSearchShortcuts();
           } else {
             this.unpinAllSearchShortcuts();
           }
           this.refresh();
           break;
         default:
           if (data.startsWith(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH)) {
             this._readDefaults();
           }
           break;
       }
       break;
   }
 }

 handlePlacesEvents(events) {
   for (const {
     itemType,
     source,
     url,
     isRemovedFromStore,
     isTagging,
     type,
   } of events) {
     switch (type) {
       case "history-cleared":
         this.frecentCache.expire();
         this.refresh();
         break;
       case "page-removed":
         if (isRemovedFromStore) {
           this.frecentCache.expire();
           this.refresh();
         }
         break;
       case "bookmark-added":
         // Skips items that are not bookmarks (like folders), about:* pages or
         // default bookmarks, added when the profile is created.
         if (
           isTagging ||
           itemType !== lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK ||
           source === lazy.PlacesUtils.bookmarks.SOURCES.IMPORT ||
           source === lazy.PlacesUtils.bookmarks.SOURCES.RESTORE ||
           source === lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP ||
           source === lazy.PlacesUtils.bookmarks.SOURCES.SYNC ||
           (!url.startsWith("http://") && !url.startsWith("https://"))
         ) {
           return;
         }

         // TODO: Add a timed delay in case many links are changed.
         this.frecentCache.expire();
         this.refresh();
         break;
       case "bookmark-removed":
         if (
           isTagging ||
           (itemType === lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK &&
             source !== lazy.PlacesUtils.bookmarks.SOURCES.IMPORT &&
             source !== lazy.PlacesUtils.bookmarks.SOURCES.RESTORE &&
             source !==
               lazy.PlacesUtils.bookmarks.SOURCES.RESTORE_ON_STARTUP &&
             source !== lazy.PlacesUtils.bookmarks.SOURCES.SYNC)
         ) {
           // TODO: Add a timed delay in case many links are changed.
           this.frecentCache.expire();
           this.refresh();
         }
         break;
     }
   }
 }

 /**
  * Returns a copied version of non-sponsored Top Sites. It will initialize
  * the component if it hasn't been already in order to set up and cache the
  * list, which will include pinned sites and search shortcuts. The number of
  * Top Sites returned is based on the number shown on New Tab due to the fact
  * it is the interface in which sites can be pinned/removed.
  *
  * @returns {Array<object>}
  *   A list of Top Sites.
  */
 async getSites() {
   await this.init();
   return structuredClone(this.#sites);
 }

 async getSearchShortcuts() {
   await this.init();
   return structuredClone(this.#searchShortcuts);
 }

 _dedupeKey(site) {
   return site && site.hostname;
 }

 /**
  * _readDefaults - sets DEFAULT_TOP_SITES
  */
 async _readDefaults({ isStartup = false } = {}) {
   this._useRemoteSetting = false;

   if (!Services.prefs.getBoolPref(REMOTE_SETTING_DEFAULTS_PREF)) {
     let sites = Services.prefs.getStringPref(DEFAULT_SITES_OVERRIDE_PREF, "");
     await this.refreshDefaults(sites, { isStartup });
     return;
   }

   // Try using default top sites from enterprise policies or tests. The pref
   // is locked when set via enterprise policy. Tests have no default sites
   // unless they set them via this pref.
   if (
     Services.prefs.prefIsLocked(DEFAULT_SITES_OVERRIDE_PREF) ||
     Cu.isInAutomation
   ) {
     let sites = Services.prefs.getStringPref(DEFAULT_SITES_OVERRIDE_PREF, "");
     await this.refreshDefaults(sites, { isStartup });
     return;
   }

   // Clear out the array of any previous defaults.
   DEFAULT_TOP_SITES.length = 0;

   // Read defaults from remote settings.
   this._useRemoteSetting = true;
   let remoteSettingData = await this._getRemoteConfig();

   for (let siteData of remoteSettingData) {
     let hostname = lazy.NewTabUtils.shortURL(siteData);
     let link = {
       isDefault: true,
       url: siteData.url,
       hostname,
       sendAttributionRequest: !!siteData.send_attribution_request,
     };
     if (siteData.url_urlbar_override) {
       link.url_urlbar = siteData.url_urlbar_override;
     }
     if (siteData.title) {
       link.label = siteData.title;
     }
     if (siteData.search_shortcut) {
       link = await this.topSiteToSearchTopSite(link);
     }
     DEFAULT_TOP_SITES.push(link);
   }

   await this.refresh({ isStartup });
 }

 async refreshDefaults(sites, { isStartup = false } = {}) {
   // Clear out the array of any previous defaults
   DEFAULT_TOP_SITES.length = 0;

   // Add default sites if any based on the pref
   if (sites) {
     for (const url of sites.split(",")) {
       const site = {
         isDefault: true,
         url,
       };
       site.hostname = lazy.NewTabUtils.shortURL(site);
       DEFAULT_TOP_SITES.push(site);
     }
   }

   await this.refresh({ isStartup });
 }

 async _getRemoteConfig(firstTime = true) {
   if (!this._remoteConfig) {
     this._remoteConfig = await lazy.RemoteSettings("top-sites");
     this._remoteConfig.on("sync", () => {
       this._readDefaults();
     });
   }

   let result = [];
   let failed = false;
   try {
     result = await this._remoteConfig.get();
   } catch (ex) {
     console.error(ex);
     failed = true;
   }
   if (!result.length) {
     console.error("Received empty top sites configuration!");
     failed = true;
   }
   // If we failed, or the result is empty, try loading from the local dump.
   if (firstTime && failed) {
     await this._remoteConfig.db.clear();
     // Now call this again.
     return this._getRemoteConfig(false);
   }

   // Sort sites based on the "order" attribute.
   result.sort((a, b) => a.order - b.order);

   result = result.filter(topsite => {
     // Filter by region.
     if (topsite.exclude_regions?.includes(lazy.Region.home)) {
       return false;
     }
     if (
       topsite.include_regions?.length &&
       !topsite.include_regions.includes(lazy.Region.home)
     ) {
       return false;
     }

     // Filter by locale.
     if (topsite.exclude_locales?.includes(Services.locale.appLocaleAsBCP47)) {
       return false;
     }
     if (
       topsite.include_locales?.length &&
       !topsite.include_locales.includes(Services.locale.appLocaleAsBCP47)
     ) {
       return false;
     }

     // Filter by experiment.
     // Exclude this top site if any of the specified experiments are running.
     if (
       topsite.exclude_experiments?.some(experimentID =>
         Services.prefs.getBoolPref(
           DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID,
           false
         )
       )
     ) {
       return false;
     }
     // Exclude this top site if none of the specified experiments are running.
     if (
       topsite.include_experiments?.length &&
       topsite.include_experiments.every(
         experimentID =>
           !Services.prefs.getBoolPref(
             DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID,
             false
           )
       )
     ) {
       return false;
     }

     return true;
   });

   return result;
 }

 /**
  * shouldFilterSearchTile - is default filtering enabled and does a given hostname match the user's default search engine?
  *
  * @param {string} hostname a top site hostname, such as "amazon" or "foo"
  * @returns {bool}
  */
 shouldFilterSearchTile(hostname) {
   if (
     Services.prefs.getBoolPref(NO_DEFAULT_SEARCH_TILE_PREF, true) &&
     (SEARCH_FILTERS.includes(hostname) ||
       hostname === this._currentSearchHostname)
   ) {
     return true;
   }
   return false;
 }

 /**
  * _maybeInsertSearchShortcuts - if the search shortcuts experiment is running,
  *                               insert search shortcuts if needed
  *
  * @param {Array} plainPinnedSites (from the pinnedSitesCache)
  * @returns {boolean} Did we insert any search shortcuts?
  */
 async _maybeInsertSearchShortcuts(plainPinnedSites) {
   // Only insert shortcuts if the experiment is running
   if (Services.prefs.getBoolPref(TOP_SITE_SEARCH_SHORTCUTS_PREF, true)) {
     // We don't want to insert shortcuts we've previously inserted
     const prevInsertedShortcuts = Services.prefs
       .getStringPref(SEARCH_SHORTCUTS_HAVE_PINNED_PREF, "")
       .split(",")
       .filter(s => s); // Filter out empty strings
     const newInsertedShortcuts = [];

     let shouldPin = this._useRemoteSetting
       ? DEFAULT_TOP_SITES.filter(s => s.searchTopSite).map(s => s.hostname)
       : Services.prefs.getStringPref(SEARCH_SHORTCUTS_ENGINES, "").split(",");
     shouldPin = shouldPin
       .map(getSearchProvider)
       .filter(s => s && s.shortURL !== this._currentSearchHostname);

     // If we've previously inserted all search shortcuts return early
     if (
       shouldPin.every(shortcut =>
         prevInsertedShortcuts.includes(shortcut.shortURL)
       )
     ) {
       return false;
     }

     const numberOfSlots =
       Services.prefs.getIntPref(TOP_SITES_ROWS_PREF, 1) *
       TOP_SITES_MAX_SITES_PER_ROW;

     // The plainPinnedSites array is populated with pinned sites at their
     // respective indices, and null everywhere else, but is not always the
     // right length
     const emptySlots = Math.max(numberOfSlots - plainPinnedSites.length, 0);
     const pinnedSites = [...plainPinnedSites].concat(
       Array(emptySlots).fill(null)
     );

     const tryToInsertSearchShortcut = async shortcut => {
       const nextAvailable = pinnedSites.indexOf(null);
       // Only add a search shortcut if the site isn't already pinned, we
       // haven't previously inserted it, there's space to pin it, and the
       // search engine is available in Firefox
       if (
         !pinnedSites.find(
           s => s && lazy.NewTabUtils.shortURL(s) === shortcut.shortURL
         ) &&
         !prevInsertedShortcuts.includes(shortcut.shortURL) &&
         nextAvailable > -1 &&
         (await checkHasSearchEngine(shortcut.keyword))
       ) {
         const site = await this.topSiteToSearchTopSite({ url: shortcut.url });
         this._pinSiteAt(site, nextAvailable);
         pinnedSites[nextAvailable] = site;
         newInsertedShortcuts.push(shortcut.shortURL);
       }
     };

     for (let shortcut of shouldPin) {
       await tryToInsertSearchShortcut(shortcut);
     }

     if (newInsertedShortcuts.length) {
       Services.prefs.setStringPref(
         SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
         prevInsertedShortcuts.concat(newInsertedShortcuts).join(",")
       );
       return true;
     }
   }

   return false;
 }

 // eslint-disable-next-line max-statements
 async getLinksWithDefaults() {
   // Clear the previous sites.
   this.#sites = [];

   const numItems =
     Services.prefs.getIntPref(TOP_SITES_ROWS_PREF, 1) *
     TOP_SITES_MAX_SITES_PER_ROW;
   const searchShortcutsExperiment = Services.prefs.getBoolPref(
     TOP_SITE_SEARCH_SHORTCUTS_PREF,
     true
   );
   // We must wait for search services to initialize in order to access default
   // search engine properties without triggering a synchronous initialization
   try {
     await Services.search.init();
   } catch {
     // We continue anyway because we want the user to see their sponsored,
     // saved, or visited shortcut tiles even if search engines are not
     // available.
   }

   // Get all frecent sites from history.
   let frecent = [];
   let cache;
   try {
     // Request can throw if executing the linkGetter inside LinksCache returns
     // a null object.
     cache = await this.frecentCache.request({
       // We need to overquery due to the top 5 alexa search + default search possibly being removed
       numItems: numItems + SEARCH_FILTERS.length + 1,
       topsiteFrecency: lazy.pageFrecencyThreshold,
     });
   } catch (ex) {
     cache = [];
   }

   for (let link of cache) {
     // The cache can contain null values.
     if (!link) {
       continue;
     }
     const hostname = lazy.NewTabUtils.shortURL(link);
     if (!this.shouldFilterSearchTile(hostname)) {
       frecent.push({
         ...(searchShortcutsExperiment
           ? await this.topSiteToSearchTopSite(link)
           : link),
         hostname,
       });
     }
   }

   // Get defaults.
   let notBlockedDefaultSites = [];
   for (let link of DEFAULT_TOP_SITES) {
     if (this.shouldFilterSearchTile(link.hostname)) {
       continue;
     }
     // Drop blocked default sites.
     if (
       lazy.NewTabUtils.blockedLinks.isBlocked({
         url: link.url,
       })
     ) {
       continue;
     }
     // If we've previously blocked a search shortcut, remove the default top site
     // that matches the hostname
     const searchProvider = getSearchProvider(lazy.NewTabUtils.shortURL(link));
     if (
       searchProvider &&
       lazy.NewTabUtils.blockedLinks.isBlocked({ url: searchProvider.url })
     ) {
       continue;
     }
     notBlockedDefaultSites.push(
       searchShortcutsExperiment
         ? await this.topSiteToSearchTopSite(link)
         : link
     );
   }

   // Get pinned links augmented with desired properties
   let plainPinned = await this.pinnedCache.request();

   // Insert search shortcuts if we need to.
   // _maybeInsertSearchShortcuts returns true if any search shortcuts are
   // inserted, meaning we need to expire and refresh the pinnedCache
   if (await this._maybeInsertSearchShortcuts(plainPinned)) {
     this.pinnedCache.expire();
     plainPinned = await this.pinnedCache.request();
   }

   const pinned = await Promise.all(
     plainPinned.map(async link => {
       if (!link) {
         return link;
       }

       // Drop pinned search shortcuts when their engine has been removed / hidden.
       if (link.searchTopSite) {
         const searchProvider = getSearchProvider(
           lazy.NewTabUtils.shortURL(link)
         );
         if (
           !searchProvider ||
           !(await checkHasSearchEngine(searchProvider.keyword))
         ) {
           return null;
         }
       }

       // Copy all properties from a frecent link and add more
       const finder = other => other.url === link.url;

       const frecentSite = frecent.find(finder);
       // If the link is a frecent site, do not copy over 'isDefault', else check
       // if the site is a default site
       const copy = Object.assign(
         {},
         frecentSite || { isDefault: !!notBlockedDefaultSites.find(finder) },
         link,
         { hostname: lazy.NewTabUtils.shortURL(link) },
         { searchTopSite: !!link.searchTopSite }
       );

       // Add in favicons if we don't already have it
       if (!copy.favicon) {
         try {
           lazy.NewTabUtils.activityStreamProvider._faviconBytesToDataURI(
             await lazy.NewTabUtils.activityStreamProvider._addFavicons([copy])
           );

           for (const prop of PINNED_FAVICON_PROPS_TO_MIGRATE) {
             copy.__sharedCache.updateLink(prop, copy[prop]);
           }
         } catch (e) {
           // Some issue with favicon, so just continue without one
         }
       }

       return copy;
     })
   );

   // Remove any duplicates from frecent and default sites
   const [, dedupedFrecent, dedupedDefaults] = this.dedupe.group(
     pinned,
     frecent,
     notBlockedDefaultSites
   );
   const dedupedUnpinned = [...dedupedFrecent, ...dedupedDefaults];

   // Remove adult sites if we need to
   const checkedAdult = lazy.FilterAdult.filter(dedupedUnpinned);

   // Insert the original pinned sites into the deduped frecent and defaults.
   let withPinned = insertPinned(checkedAdult, pinned);
   // Remove excess items.
   withPinned = withPinned.slice(0, numItems);

   // Now, get a tippy top icon or a rich icon for every item.
   for (const link of withPinned) {
     if (link) {
       if (link.searchTopSite && !link.isDefault) {
         this._tippyTopProvider.processSite(link);
       } else {
         this._fetchIcon(link);
       }

       // Remove internal properties that might be updated after dispatch
       delete link.__sharedCache;

       // Indicate that these links should get a frecency bonus when clicked
       link.typedBonus = true;
     }
   }

   this.#sites = withPinned;

   return withPinned;
 }

 /**
  * Refresh the top sites data for content.
  *
  * @param {object} options
  * @param {bool} options.isStartup Being called while TopSitesFeed is initting.
  */
 async refresh(options = {}) {
   // Avoiding refreshing if it's already happening.
   if (this._refreshing) {
     return;
   }
   if (!this._startedUp && !options.isStartup) {
     // Initial refresh still pending.
     return;
   }
   this._refreshing = true;
   this._startedUp = true;

   if (!this._tippyTopProvider.initialized) {
     await this._tippyTopProvider.init();
   }

   await this.getLinksWithDefaults();
   this._refreshing = false;
   Services.obs.notifyObservers(null, "topsites-refreshed", options.isStartup);
 }

 async updateCustomSearchShortcuts(isStartup = false) {
   if (
     !Services.prefs.getBoolPref(
       "browser.newtabpage.activity-stream.improvesearch.noDefaultSearchTile",
       true
     )
   ) {
     return;
   }

   if (!this._tippyTopProvider.initialized) {
     await this._tippyTopProvider.init();
   }

   // Populate the state with available search shortcuts
   let searchShortcuts = [];
   for (const engine of await Services.search.getAppProvidedEngines()) {
     const shortcut = CUSTOM_SEARCH_SHORTCUTS.find(s =>
       engine.aliases.includes(s.keyword)
     );
     if (shortcut) {
       let clone = { ...shortcut };
       this._tippyTopProvider.processSite(clone);
       searchShortcuts.push(clone);
     }
   }

   // TODO: Determine what the purpose of this is.
   this.#searchShortcuts = searchShortcuts;

   Services.obs.notifyObservers(
     null,
     "topsites-updated-custom-search-shortcuts",
     isStartup
   );
 }

 async topSiteToSearchTopSite(site) {
   const searchProvider = getSearchProvider(lazy.NewTabUtils.shortURL(site));
   if (
     !searchProvider ||
     !(await checkHasSearchEngine(searchProvider.keyword))
   ) {
     return site;
   }
   return {
     ...site,
     searchTopSite: true,
     label: searchProvider.keyword,
   };
 }

 /**
  * Get an image for the link preferring tippy top, or rich favicon.
  */
 async _fetchIcon(link) {
   // Nothing to do if we already have a rich icon from the page
   if (link.favicon && link.faviconSize >= MIN_FAVICON_SIZE) {
     return;
   }

   // Nothing more to do if we can use a default tippy top icon
   this._tippyTopProvider.processSite(link);
   if (link.tippyTopIcon) {
     return;
   }

   // Make a request for a better icon
   this._requestRichIcon(link.url);
 }

 _requestRichIcon(url) {
   this._faviconProvider.fetchIcon(url);
 }

 /**
  * Inform others that top sites data has been updated due to pinned changes.
  */
 _broadcastPinnedSitesUpdated() {
   // Pinned data changed, so make sure we get latest
   this.pinnedCache.expire();

   // Refresh to trigger deduping, etc.
   this.refresh();
 }

 /**
  * Pin a site at a specific position saving only the desired keys.
  *
  * @param label {string} User set string of custom site name
  */
 // To refactor in Bug 1891997
 /* eslint-enable jsdoc/check-param-names */
 async _pinSiteAt({ label, url, searchTopSite }, index) {
   const toPin = { url };
   if (label) {
     toPin.label = label;
   }
   if (searchTopSite) {
     toPin.searchTopSite = searchTopSite;
   }
   lazy.NewTabUtils.pinnedLinks.pin(toPin, index);
 }

 /**
  * Handle a pin action of a site to a position.
  */
 async pin(action) {
   let { site, index } = action.data;
   index = this._adjustPinIndexForSponsoredLinks(site, index);
   // If valid index provided, pin at that position
   if (index >= 0) {
     await this._pinSiteAt(site, index);
     this._broadcastPinnedSitesUpdated();
   } else {
     // Bug 1458658. If the top site is being pinned from an 'Add a Top Site' option,
     // then we want to make sure to unblock that link if it has previously been
     // blocked. We know if the site has been added because the index will be -1.
     if (index === -1) {
       lazy.NewTabUtils.blockedLinks.unblock({ url: site.url });
       this.frecentCache.expire();
     }
     this.insert(action);
   }
 }

 /**
  * Handle an unpin action of a site.
  */
 unpin(action) {
   const { site } = action.data;
   lazy.NewTabUtils.pinnedLinks.unpin(site);
   this._broadcastPinnedSitesUpdated();
 }

 unpinAllSearchShortcuts() {
   Services.prefs.clearUserPref(SEARCH_SHORTCUTS_HAVE_PINNED_PREF);
   for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) {
     if (pinnedLink && pinnedLink.searchTopSite) {
       lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink);
     }
   }
   this.pinnedCache.expire();
 }

 _unpinSearchShortcut(vendor) {
   for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) {
     if (
       pinnedLink &&
       pinnedLink.searchTopSite &&
       lazy.NewTabUtils.shortURL(pinnedLink) === vendor
     ) {
       lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink);
       this.pinnedCache.expire();

       const prevInsertedShortcuts = Services.prefs.getStringPref(
         SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
         ""
       );
       Services.prefs.setStringPref(
         SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
         prevInsertedShortcuts.filter(s => s !== vendor).join(",")
       );
       break;
     }
   }
 }

 /**
  * Reduces the given pinning index by the number of preceding sponsored
  * sites, to accomodate for sponsored sites pushing pinned ones to the side,
  * effectively increasing their index again.
  */
 _adjustPinIndexForSponsoredLinks(site, index) {
   if (!this.#sites) {
     return index;
   }
   // Adjust insertion index for sponsored sites since their position is
   // fixed.
   let adjustedIndex = index;
   for (let i = 0; i < index; i++) {
     const link = this.#sites[i];
     if (link && link.sponsored_position && this.#sites[i]?.url !== site.url) {
       adjustedIndex--;
     }
   }
   return adjustedIndex;
 }

 /**
  * Insert a site to pin at a position shifting over any other pinned sites.
  */
 _insertPin(site, originalIndex, draggedFromIndex) {
   let index = this._adjustPinIndexForSponsoredLinks(site, originalIndex);

   // Don't insert any pins past the end of the visible top sites. Otherwise,
   // we can end up with a bunch of pinned sites that can never be unpinned again
   // from the UI.
   const topSitesCount =
     Services.prefs.getIntPref(TOP_SITES_ROWS_PREF, 1) *
     TOP_SITES_MAX_SITES_PER_ROW;
   if (index >= topSitesCount) {
     return;
   }

   let pinned = lazy.NewTabUtils.pinnedLinks.links;
   if (!pinned[index]) {
     this._pinSiteAt(site, index);
   } else {
     pinned[draggedFromIndex] = null;
     // Find the hole to shift the pinned site(s) towards. We shift towards the
     // hole left by the site being dragged.
     let holeIndex = index;
     const indexStep = index > draggedFromIndex ? -1 : 1;
     while (pinned[holeIndex]) {
       holeIndex += indexStep;
     }
     if (holeIndex >= topSitesCount || holeIndex < 0) {
       // There are no holes, so we will effectively unpin the last slot and shifting
       // towards it. This only happens when adding a new top site to an already
       // fully pinned grid.
       holeIndex = topSitesCount - 1;
     }

     // Shift towards the hole.
     const shiftingStep = holeIndex > index ? -1 : 1;
     while (holeIndex !== index) {
       const nextIndex = holeIndex + shiftingStep;
       this._pinSiteAt(pinned[nextIndex], holeIndex);
       holeIndex = nextIndex;
     }
     this._pinSiteAt(site, index);
   }
 }

 /**
  * Handle an insert (drop/add) action of a site.
  */
 async insert(action) {
   let { index } = action.data;
   // Treat invalid pin index values (e.g., -1, undefined) as insert in the first position
   if (!(index > 0)) {
     index = 0;
   }

   // Inserting a top site pins it in the specified slot, pushing over any link already
   // pinned in the slot (unless it's the last slot, then it replaces).
   this._insertPin(
     action.data.site,
     index,
     action.data.draggedFromIndex !== undefined
       ? action.data.draggedFromIndex
       : Services.prefs.getIntPref(TOP_SITES_ROWS_PREF, 1) *
           TOP_SITES_MAX_SITES_PER_ROW
   );

   this._broadcastPinnedSitesUpdated();
 }

 updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }) {
   // Unpin the deletedShortcuts.
   deletedShortcuts.forEach(({ url }) => {
     lazy.NewTabUtils.pinnedLinks.unpin({ url });
   });

   // Pin the addedShortcuts.
   const numberOfSlots =
     Services.prefs.getIntPref(TOP_SITES_ROWS_PREF, 1) *
     TOP_SITES_MAX_SITES_PER_ROW;
   addedShortcuts.forEach(shortcut => {
     // Find first hole in pinnedLinks.
     let index = lazy.NewTabUtils.pinnedLinks.links.findIndex(link => !link);
     if (
       index < 0 &&
       lazy.NewTabUtils.pinnedLinks.links.length + 1 < numberOfSlots
     ) {
       // pinnedLinks can have less slots than the total available.
       index = lazy.NewTabUtils.pinnedLinks.links.length;
     }
     if (index >= 0) {
       lazy.NewTabUtils.pinnedLinks.pin(shortcut, index);
     } else {
       // No slots available, we need to do an insert in first slot and push over other pinned links.
       this._insertPin(shortcut, 0, numberOfSlots);
     }
   });

   this._broadcastPinnedSitesUpdated();
 }
}

/**
* insertPinned - Inserts pinned links in their specified slots
*
* @param {Array} links list of links
* @param {Array} pinned list of pinned links
* @returns {Array} resulting list of links with pinned links inserted
*/
export function insertPinned(links, pinned) {
 // Remove any pinned links
 const pinnedUrls = pinned.map(link => link && link.url);
 let newLinks = links.filter(link =>
   link ? !pinnedUrls.includes(link.url) : false
 );
 newLinks = newLinks.map(link => {
   if (link && link.isPinned) {
     delete link.isPinned;
     delete link.pinIndex;
   }
   return link;
 });

 // Then insert them in their specified location
 pinned.forEach((val, index) => {
   if (!val) {
     return;
   }
   let link = Object.assign({}, val, { isPinned: true, pinIndex: index });
   if (index > newLinks.length) {
     newLinks[index] = link;
   } else {
     newLinks.splice(index, 0, link);
   }
 });

 return newLinks;
}

/**
* FaviconProvider class handles the retrieval and management of favicons
* for TopSites.
*/
export class FaviconProvider {
 constructor() {
   this._queryForRedirects = new Set();
 }

 /**
  * fetchIcon attempts to fetch a rich icon for the given url from two sources.
  * First, it looks up the tippy top feed, if it's still missing, then it queries
  * the places for rich icon with its most recent visit in order to deal with
  * the redirected visit. See Bug 1421428 for more details.
  */
 async fetchIcon(url) {
   // Avoid initializing and fetching icons if prefs are turned off
   if (!this.shouldFetchIcons) {
     return;
   }

   const site = await this.getSite(getDomain(url));
   if (!site) {
     if (!this._queryForRedirects.has(url)) {
       this._queryForRedirects.add(url);
       Services.tm.idleDispatchToMainThread(() =>
         this.fetchIconFromRedirects(url)
       );
     }
     return;
   }

   let iconUri = Services.io.newURI(site.image_url);
   // The #tippytop is to be able to identify them for telemetry.
   iconUri = iconUri.mutate().setRef("tippytop").finalize();
   await this.#setFaviconForPage(Services.io.newURI(url), iconUri);
 }

 /**
  * Get the site tippy top data from Remote Settings.
  */
 async getSite(domain) {
   const sites = await this.tippyTop.get({
     filters: { domain },
     syncIfEmpty: false,
   });
   return sites.length ? sites[0] : null;
 }

 /**
  * Get the tippy top collection from Remote Settings.
  */
 get tippyTop() {
   if (!this._tippyTop) {
     this._tippyTop = lazy.RemoteSettings("tippytop");
   }
   return this._tippyTop;
 }

 /**
  * Determine if we should be fetching and saving icons.
  */
 get shouldFetchIcons() {
   return Services.prefs.getBoolPref("browser.chrome.site_icons");
 }

 /**
  * Get favicon info (uri and size) for a uri from Places.
  *
  * @param {nsIURI} uri
  *        Page to check for favicon data
  * @returns {object}
  *        Favicon info object. If there is no data in DB, return null.
  */
 async getFaviconInfo(uri) {
   let favicon = await lazy.PlacesUtils.favicons.getFaviconForPage(
     uri,
     lazy.NewTabUtils.activityStreamProvider.THUMB_FAVICON_SIZE
   );
   return favicon
     ? { iconUri: favicon.uri, faviconSize: favicon.width }
     : null;
 }

 /**
  * Fetch favicon for a url by following its redirects in Places.
  *
  * This can improve the rich icon coverage for Top Sites since Places only
  * associates the favicon to the final url if the original one gets redirected.
  * Note this is not an urgent request, hence it is dispatched to the main
  * thread idle handler to avoid any possible performance impact.
  */
 async fetchIconFromRedirects(url) {
   const visitPaths = await this.#fetchVisitPaths(url);
   if (visitPaths.length > 1) {
     const lastVisit = visitPaths.pop();
     const redirectedUri = Services.io.newURI(lastVisit.url);
     const iconInfo = await this.getFaviconInfo(redirectedUri);
     if (iconInfo?.faviconSize >= MIN_FAVICON_SIZE) {
       try {
         await lazy.PlacesUtils.favicons.tryCopyFavicons(
           redirectedUri,
           Services.io.newURI(url),
           lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE
         );
       } catch (ex) {
         console.error(`Failed to copy favicon [${ex}]`);
       }
     }
   }
 }

 /**
  * Get favicon data for given URL from network.
  *
  * @param {nsIURI} faviconURI
  *        nsIURI for the favicon.
  * @returns {nsIURI} data URL
  */
 async getFaviconDataURLFromNetwork(faviconURI) {
   let channel = lazy.NetUtil.newChannel({
     uri: faviconURI,
     loadingPrincipal: Services.scriptSecurityManager.getSystemPrincipal(),
     securityFlags:
       Ci.nsILoadInfo.SEC_REQUIRE_CORS_INHERITS_SEC_CONTEXT |
       Ci.nsILoadInfo.SEC_ALLOW_CHROME |
       Ci.nsILoadInfo.SEC_DISALLOW_SCRIPT,
     contentPolicyType: Ci.nsIContentPolicy.TYPE_INTERNAL_IMAGE_FAVICON,
   });

   let resolver = Promise.withResolvers();

   lazy.NetUtil.asyncFetch(channel, async (input, status, request) => {
     if (!Components.isSuccessCode(status)) {
       resolver.resolve();
       return;
     }

     try {
       let data = lazy.NetUtil.readInputStream(input, input.available());
       let { contentType } = request.QueryInterface(Ci.nsIChannel);
       input.close();

       let buffer = new Uint8ClampedArray(data);
       let blob = new Blob([buffer], { type: contentType });
       let dataURL = await new Promise((resolve, reject) => {
         let reader = new FileReader();
         reader.addEventListener("load", () => resolve(reader.result));
         reader.addEventListener("error", reject);
         reader.readAsDataURL(blob);
       });
       resolver.resolve(Services.io.newURI(dataURL));
     } catch (e) {
       resolver.reject(e);
     }
   });

   return resolver.promise;
 }

 /**
  * Set favicon for page.
  *
  * @param {nsIURI} pageURI
  * @param {nsIURI} faviconURI
  */
 async #setFaviconForPage(pageURI, faviconURI) {
   try {
     // If the given faviconURI is data URL, set it as is.
     if (faviconURI.schemeIs("data")) {
       lazy.PlacesUtils.favicons
         .setFaviconForPage(pageURI, faviconURI, faviconURI)
         .catch(console.error);
       return;
     }

     // Try to find the favicon data from DB.
     const faviconInfo = await this.getFaviconInfo(pageURI);
     if (faviconInfo?.faviconSize) {
       // As valid favicon data is already stored for the page,
       // we don't have to update.
       return;
     }

     // Otherwise, fetch from network.
     lazy.PlacesUtils.favicons
       .setFaviconForPage(
         pageURI,
         faviconURI,
         await this.getFaviconDataURLFromNetwork(faviconURI)
       )
       .catch(console.error);
   } catch (ex) {
     console.error(`Failed to set favicon for page:${ex}`);
   }
 }

 /**
  * Fetches visit paths for a given URL from its most recent visit in Places.
  *
  * Note that this includes the URL itself as well as all the following
  * permenent&temporary redirected URLs if any.
  *
  * @param {string} url
  *        a URL string
  *
  * @returns {Array} Returns an array containing objects as
  *   {int}    visit_id: ID of the visit in moz_historyvisits.
  *   {String} url: URL of the redirected URL.
  */
 async #fetchVisitPaths(url) {
   const query = `
   WITH RECURSIVE path(visit_id)
   AS (
     SELECT v.id
     FROM moz_places h
     JOIN moz_historyvisits v
       ON v.place_id = h.id
     WHERE h.url_hash = hash(:url) AND h.url = :url
       AND v.visit_date = h.last_visit_date

     UNION

     SELECT id
     FROM moz_historyvisits
     JOIN path
       ON visit_id = from_visit
     WHERE visit_type IN
       (${lazy.PlacesUtils.history.TRANSITIONS.REDIRECT_PERMANENT},
        ${lazy.PlacesUtils.history.TRANSITIONS.REDIRECT_TEMPORARY})
   )
   SELECT visit_id, (
     SELECT (
       SELECT url
       FROM moz_places
       WHERE id = place_id)
     FROM moz_historyvisits
     WHERE id = visit_id) AS url
   FROM path
 `;

   const visits =
     await lazy.NewTabUtils.activityStreamProvider.executePlacesQuery(query, {
       columns: ["visit_id", "url"],
       params: { url },
     });
   return visits;
 }
}

export const TopSites = new _TopSites();