browser_all_files_referenced.js (39564B)
1 /* Any copyright is dedicated to the Public Domain. 2 * http://creativecommons.org/publicdomain/zero/1.0/ */ 3 4 // Note to run this test similar to try server, you need to run: 5 // ./mach package 6 // ./mach mochitest --appname dist <path to test> 7 8 // Slow on asan builds. 9 requestLongerTimeout(5); 10 11 var isDevtools = SimpleTest.harnessParameters.subsuite == "devtools"; 12 13 // This list should contain only path prefixes. It is meant to stop the test 14 // from reporting things that *are* referenced, but for which the test can't 15 // find any reference because the URIs are constructed programatically. 16 // If you need to allowlist specific files, please use the 'allowlist' object. 17 var gExceptionPaths = [ 18 "resource://app/defaults/settings/blocklists/", 19 "resource://app/defaults/settings/security-state/", 20 "resource://app/defaults/settings/main/", 21 "resource://app/defaults/preferences/", 22 "resource://gre/modules/commonjs/", 23 "resource://gre/defaults/pref/", 24 25 // These chrome resources are referenced using relative paths from JS files. 26 "chrome://global/content/certviewer/components/", 27 28 // https://github.com/mozilla/activity-stream/issues/3053 29 "chrome://activity-stream/content/data/content/tippytop/images/", 30 "chrome://activity-stream/content/data/content/tippytop/favicons/", 31 // These resources are referenced by messages delivered through Remote Settings 32 "chrome://activity-stream/content/data/content/assets/mobile-download-qr-new-user-cn.svg", 33 "chrome://activity-stream/content/data/content/assets/mobile-download-qr-existing-user-cn.svg", 34 "chrome://activity-stream/content/data/content/assets/mr-amo-collection.svg", 35 "chrome://activity-stream/content/data/content/assets/person-typing.svg", 36 "chrome://activity-stream/content/data/content/assets/tabs-side-zap-transparent.svg", 37 "chrome://activity-stream/content/data/content/assets/tabs-top-zap-transparent.svg", 38 "chrome://activity-stream/content/data/content/assets/nuo-taborientation.svg", 39 "chrome://activity-stream/content/data/content/assets/euo-tab-orientation.svg", 40 "chrome://activity-stream/content/data/content/assets/euo-chatbot.svg", 41 "chrome://browser/content/assets/moz-vpn.svg", 42 "chrome://browser/content/assets/vpn-logo.svg", 43 "chrome://browser/content/assets/focus-promo.png", 44 "chrome://browser/content/assets/klar-qr-code.svg", 45 "chrome://browser/content/asrouter/assets/fox-with-box-on-cloud.svg", 46 "chrome://browser/content/asrouter/assets/fox-with-devices.svg", 47 "chrome://browser/content/asrouter/assets/fox-with-locked-box.svg", 48 "chrome://browser/content/asrouter/assets/fox-with-mobile.svg", 49 "chrome://browser/content/asrouter/assets/desktop-to-mobile-banner.svg", 50 "chrome://browser/content/asrouter/assets/desktop-to-mobile-non-eu-QR.svg", 51 "chrome://browser/content/asrouter/assets/desktop-to-mobile-eu-QR.svg", 52 53 // toolkit/components/pdfjs/content/build/pdf.js 54 "resource://pdf.js/web/images/", 55 // This file is only loaded in using a dynamic import in pdf.js in case wasm 56 // is not available. 57 "resource://pdf.js/web/wasm/openjpeg_nowasm_fallback.js", 58 59 // Exclude the form autofill path that has been moved out of the extensions to 60 // toolkit, see bug 1691821. 61 "resource://gre-resources/autofill/", 62 // Localization file added programatically in FormAutofillUtils.sys.mjs 63 "resource://gre/localization/en-US/toolkit/formautofill", 64 65 // Exclude all search-extensions because they aren't referenced by filename 66 "resource://search-extensions/", 67 68 // Exclude all services-automation because they are used through webdriver 69 "resource://gre/modules/services-automation/", 70 71 // Paths from this folder are constructed in NetErrorParent.sys.mjs based on 72 // the type of cert or net error the user is encountering. 73 "chrome://global/content/neterror/supportpages/", 74 75 // Points to theme preview images, which are defined in browser/ but only used 76 // in toolkit/mozapps/extensions/content/aboutaddons.js. 77 "resource://usercontext-content/builtin-themes/", 78 79 // Page data schemas are referenced programmatically. 80 "chrome://browser/content/pagedata/schemas/", 81 82 // Nimbus schemas are referenced programmatically. 83 "resource://nimbus/schemas/", 84 85 // Normandy schemas are referenced programmatically. 86 "resource://normandy/schemas/", 87 88 // ASRouter schemas are referenced programmatically. 89 "chrome://browser/content/asrouter/schemas/", 90 91 // Localization file added programatically in FeatureCallout.sys.mjs 92 "resource://app/localization/en-US/browser/featureCallout.ftl", 93 94 // Localization file added programatically in ContentAnalysis.sys.mjs 95 "resource://gre/localization/en-US/toolkit/contentanalysis/", 96 97 // CSS files are referenced inside JS in an html template 98 "chrome://browser/content/aboutlogins/components/", 99 100 // Strip on Share parameter lists 101 "chrome://global/content/antitracking/", 102 103 // CSS file is referenced inside JS in login-form.mjs 104 "chrome://global/content/megalist/LoginFormComponent/", 105 106 // The ONNX runtime picks files to run programmaticaly 107 "chrome://global/content/ml/", 108 109 // The profile avatars are directly referenced. 110 "chrome://browser/content/profiles/assets/", 111 112 // The picture-in-picture add-on. 113 "resource://builtin-addons/pictureinpicture/", 114 115 // The formautofill add-on. 116 "resource://builtin-addons/formautofill/", 117 118 // The webcompat add-on. 119 "resource://builtin-addons/webcompat/", 120 121 // The newtab add-on 122 "resource://builtin-addons/newtab/", 123 "resource://newtab/", 124 "chrome://newtab/", 125 126 // UniFFI test files. 127 "moz-src:///toolkit/components/uniffi-bindgen-gecko-js/tests/generated/", 128 129 // Used for Market suggestions on the urlbar. This is specified from Remote 130 // Settings. 131 "chrome://browser/skin/illustrations/market-opt-in.svg", 132 133 // Used for Yelp realtime suggestions on the urlbar. This is specified from 134 // Remote Settings. 135 "chrome://browser/skin/illustrations/yelpRealtime-opt-in.svg", 136 ]; 137 138 // These are not part of the omni.ja file, so we find them only when running 139 // the test on a non-packaged build. 140 if (AppConstants.platform == "macosx") { 141 gExceptionPaths.push("resource://gre/res/cursors/"); 142 gExceptionPaths.push("resource://gre/res/touchbar/"); 143 } 144 145 if (AppConstants.MOZ_BACKGROUNDTASKS) { 146 // These preferences are active only when we're in background task mode. 147 gExceptionPaths.push("resource://gre/defaults/backgroundtasks/"); 148 gExceptionPaths.push("resource://app/defaults/backgroundtasks/"); 149 // `BackgroundTask_*.sys.mjs` are loaded at runtime by `app --backgroundtask id ...`. 150 gExceptionPaths.push("resource://gre/modules/backgroundtasks/"); 151 gExceptionPaths.push("resource://app/modules/backgroundtasks/"); 152 } 153 154 // Each allowlist entry should have a comment indicating which file is 155 // referencing the listed file in a way that the test can't detect, or a 156 // bug number to remove or use the file if it is indeed currently unreferenced. 157 var allowlist = [ 158 // security/manager/pki/resources/content/device_manager.js 159 { file: "chrome://pippki/content/load_device.xhtml" }, 160 161 // Intentionally unreferenced, see bug 1941134 162 { file: "resource://gre/res/designmode.css" }, 163 { file: "resource://gre/res/EditorOverride.css" }, 164 165 // The l10n build system can't package string files only for some platforms. 166 // See bug 1339424 for why this is hard to fix. 167 { 168 file: "chrome://global/locale/fallbackMenubar.properties", 169 platforms: ["linux", "win"], 170 }, 171 { 172 file: "resource://gre/localization/en-US/toolkit/printing/printDialogs.ftl", 173 platforms: ["linux", "macosx"], 174 }, 175 176 // This file is referenced by the build system to generate the 177 // Firefox .desktop entry. See bug 1824327 (and perhaps bug 1526672) 178 { 179 file: "resource://app/localization/en-US/browser/linuxDesktopEntry.ftl", 180 }, 181 182 // devtools/client/inspector/bin/dev-server.js 183 { 184 file: "chrome://devtools/content/inspector/markup/markup.xhtml", 185 isFromDevTools: true, 186 }, 187 188 // SpiderMonkey parser API, currently unused in browser/ and toolkit/ 189 { file: "resource://gre/modules/reflect.sys.mjs" }, 190 191 // extensions/pref/autoconfig/src/nsReadConfig.cpp 192 { file: "resource://gre/defaults/autoconfig/prefcalls.js" }, 193 194 // browser/components/preferences/moreFromMozilla.js 195 // These files URLs are constructed programatically at run time. 196 { 197 file: "chrome://browser/content/preferences/more-from-mozilla-qr-code-simple.svg", 198 }, 199 { 200 file: "chrome://browser/content/preferences/more-from-mozilla-qr-code-simple-cn.svg", 201 }, 202 203 { file: "resource://gre/greprefs.js" }, 204 205 // toolkit/mozapps/extensions/AddonContentPolicy.cpp 206 { file: "resource://gre/localization/en-US/toolkit/global/cspErrors.ftl" }, 207 208 // toolkit/components/antitracking/bouncetrackingprotection/BounceTrackingProtection.cpp 209 { file: "resource://gre/localization/en-US/toolkit/global/antiTracking.ftl" }, 210 211 // The l10n build system can't package string files only for some platforms. 212 { 213 file: "resource://gre/chrome/en-US/locale/en-US/global-platform/mac/accessible.properties", 214 platforms: ["linux", "win"], 215 }, 216 { 217 file: "resource://gre/chrome/en-US/locale/en-US/global-platform/mac/platformKeys.properties", 218 platforms: ["linux", "win"], 219 }, 220 { 221 file: "resource://gre/chrome/en-US/locale/en-US/global-platform/unix/accessible.properties", 222 platforms: ["macosx", "win"], 223 }, 224 { 225 file: "resource://gre/chrome/en-US/locale/en-US/global-platform/unix/platformKeys.properties", 226 platforms: ["macosx", "win"], 227 }, 228 { 229 file: "resource://gre/chrome/en-US/locale/en-US/global-platform/win/accessible.properties", 230 platforms: ["linux", "macosx"], 231 }, 232 { 233 file: "resource://gre/chrome/en-US/locale/en-US/global-platform/win/platformKeys.properties", 234 platforms: ["linux", "macosx"], 235 }, 236 237 // Files from upstream library 238 { file: "resource://pdf.js/web/debugger.mjs" }, 239 { file: "resource://pdf.js/web/debugger.css" }, 240 241 // File from the ipp-activator add-on 242 { file: "resource://builtin-addons/ipp-activator/breakages/tab.json" }, 243 244 // Starting from here, files in the allowlist are bugs that need fixing. 245 // Bug 1339424 (wontfix?) 246 { 247 file: "chrome://browser/locale/taskbar.properties", 248 platforms: ["linux", "macosx"], 249 }, 250 // Bug 1348559 251 { file: "chrome://pippki/content/resetpassword.xhtml" }, 252 // Bug 1337345 253 { file: "resource://gre/modules/Manifest.sys.mjs" }, 254 // Bug 1494170 255 // (The references to these files are dynamically generated, so the test can't 256 // find the references) 257 { 258 file: "chrome://devtools/skin/images/aboutdebugging-firefox-aurora.svg", 259 isFromDevTools: true, 260 }, 261 { 262 file: "chrome://devtools/skin/images/aboutdebugging-firefox-beta.svg", 263 isFromDevTools: true, 264 }, 265 { 266 file: "chrome://devtools/skin/images/aboutdebugging-firefox-release.svg", 267 isFromDevTools: true, 268 }, 269 { file: "chrome://devtools/skin/images/next.svg", isFromDevTools: true }, 270 271 // Bug 1526672 272 { 273 file: "resource://app/localization/en-US/browser/touchbar/touchbar.ftl", 274 platforms: ["linux", "win"], 275 }, 276 277 // dom/media/mediacontrol/MediaControlService.cpp 278 { file: "resource://gre/localization/en-US/dom/media.ftl" }, 279 280 // dom/xml/nsXMLPrettyPrinter.cpp 281 { file: "resource://gre/localization/en-US/dom/XMLPrettyPrint.ftl" }, 282 283 // tookit/mozapps/update/BackgroundUpdate.sys.mjs 284 { 285 file: "resource://gre/localization/en-US/toolkit/updates/backgroundupdate.ftl", 286 }, 287 288 // Bug 1713242 - referenced by aboutThirdParty.html which is only for Windows 289 { 290 file: "resource://gre/localization/en-US/toolkit/about/aboutThirdParty.ftl", 291 platforms: ["linux", "macosx"], 292 }, 293 // Bug 1854618 - referenced by aboutWebauthn.html which is only for Linux and Mac 294 { 295 file: "resource://gre/localization/en-US/toolkit/about/aboutWebauthn.ftl", 296 platforms: ["win", "android"], 297 }, 298 // Bug 1973834 - referenced by aboutWindowsMessages.html which is only for Windows 299 { 300 file: "resource://gre/localization/en-US/toolkit/about/aboutWindowsMessages.ftl", 301 platforms: ["linux", "macosx"], 302 }, 303 // Bug 1721741: 304 // (The references to these files are dynamically generated, so the test can't 305 // find the references) 306 { file: "chrome://browser/content/screenshots/copied-notification.svg" }, 307 308 // toolkit/xre/MacRunFromDmgUtils.mm 309 { file: "resource://gre/localization/en-US/toolkit/global/run-from-dmg.ftl" }, 310 311 // Referenced programmatically 312 { file: "chrome://browser/content/backup/BackupManifest.1.schema.json" }, 313 { file: "chrome://browser/content/backup/ArchiveJSONBlock.1.schema.json" }, 314 315 // Bug 1733498 - Migrate necko errors l10n strings from .properties to Fluent 316 { 317 file: "resource://gre/localization/en-US/netwerk/necko.ftl", 318 }, 319 320 // dom/xslt/xslt/txMozillaXSLTProcessor.cpp 321 { file: "resource://gre/localization/en-US/dom/xslt.ftl" }, 322 323 // A QA and dev debug tool. 324 { file: "chrome://browser/content/places/interactionsViewer.html" }, 325 326 // Bug 1984409: We're doing backups to cloud-synced locations first. We'll do local backups eventually, 327 // and this file will be needed for that. 328 { 329 file: "resource://app/modules/backup/CookiesBackupResource.sys.mjs", 330 }, 331 332 // Bug 2000945 - Move query intent detection to AI-window r?mardak (backed out due to unused file) 333 { 334 file: "moz-src:///browser/components/aiwindow/models/IntentClassifier.sys.mjs", 335 }, 336 // Bug 2005768 - Insights scheduler for generation from history 337 // Bug 2007939 - Rename "insights" to "memories" 338 { 339 file: "moz-src:///browser/components/aiwindow/models/memories/MemoriesHistoryScheduler.sys.mjs", 340 }, 341 // Bug 2006090 - Insight updation - Day 0 and incremental updates from Chat history 342 // Bug 2007939 - Rename "insights" to "memories" 343 { 344 file: "moz-src:///browser/components/aiwindow/models/memories/MemoriesConversationScheduler.sys.mjs", 345 }, 346 // Bug 2006433 - Implement conversation starter/followup inference 347 { 348 file: "moz-src:///browser/components/aiwindow/models/ConversationSuggestions.sys.mjs", 349 }, 350 // Bug 1996315: QR code generation modules 351 { 352 file: "moz-src:///browser/components/qrcode/QRCodeGenerator.sys.mjs", 353 }, 354 { 355 file: "moz-src:///browser/components/qrcode/QRCodeWorker.sys.mjs", 356 }, 357 ]; 358 359 if (AppConstants.NIGHTLY_BUILD) { 360 allowlist.push( 361 // A debug tool that is only available in Nightly builds, and is accessed 362 // directly by developers via the chrome URI (bug 1888491) 363 { file: "chrome://browser/content/backup/debug.html" } 364 ); 365 } 366 367 if (AppConstants.platform != "win") { 368 // toolkit/mozapps/defaultagent/Notification.cpp 369 // toolkit/mozapps/defaultagent/ScheduledTask.cpp 370 // toolkit/mozapps/defaultagent/BackgroundTask_defaultagent.sys.mjs 371 // Bug 1854425 - referenced by default browser agent which is not detected 372 allowlist.push({ 373 file: "resource://app/localization/en-US/browser/backgroundtasks/defaultagent.ftl", 374 }); 375 376 if (AppConstants.NIGHTLY_BUILD) { 377 // This path is refereneced in nsFxrCommandLineHandler.cpp, which is only 378 // compiled in Windows. This path is allowed so that non-Windows builds 379 // can access the FxR UI via --chrome rather than --fxr (which includes VR- 380 // specific functionality) 381 allowlist.push({ file: "chrome://fxr/content/fxrui.html" }); 382 } 383 } 384 385 if (AppConstants.platform == "android") { 386 // The l10n build system can't package string files only for some platforms. 387 // Referenced by aboutGlean.html 388 allowlist.push({ 389 file: "resource://gre/localization/en-US/toolkit/about/aboutGlean.ftl", 390 }); 391 } 392 393 if (AppConstants.MOZ_UPDATE_AGENT && !AppConstants.MOZ_BACKGROUNDTASKS) { 394 // Task scheduling is only used for background updates right now. 395 allowlist.push({ 396 file: "resource://gre/modules/TaskScheduler.sys.mjs", 397 }); 398 } 399 400 allowlist = new Set( 401 allowlist 402 .filter( 403 item => 404 "isFromDevTools" in item == isDevtools && 405 (!item.skipUnofficial || !AppConstants.MOZILLA_OFFICIAL) && 406 (!item.platforms || item.platforms.includes(AppConstants.platform)) 407 ) 408 .map(item => item.file) 409 ); 410 411 const ignorableAllowlist = new Set([ 412 // The following files are outside of the omni.ja file, so we only catch them 413 // when testing on a non-packaged build. 414 415 // dom/media/gmp/GMPParent.cpp 416 "resource://gre/gmp-clearkey/0.1/manifest.json", 417 ]); 418 for (let entry of ignorableAllowlist) { 419 allowlist.add(entry); 420 } 421 422 if (!isDevtools) { 423 // services/sync/modules/service.sys.mjs 424 for (let module of [ 425 "addons.sys.mjs", 426 "bookmarks.sys.mjs", 427 "forms.sys.mjs", 428 "history.sys.mjs", 429 "passwords.sys.mjs", 430 "prefs.sys.mjs", 431 "tabs.sys.mjs", 432 "extension-storage.sys.mjs", 433 ]) { 434 allowlist.add("resource://services-sync/engines/" + module); 435 } 436 // resource://devtools/shared/worker/loader.js, 437 // resource://devtools/shared/loader/builtin-modules.js 438 if (!AppConstants.ENABLE_WEBDRIVER) { 439 allowlist.add("resource://gre/modules/jsdebugger.sys.mjs"); 440 } 441 } 442 443 if (AppConstants.MOZ_CODE_COVERAGE) { 444 allowlist.add( 445 "chrome://remote/content/marionette/PerTestCoverageUtils.sys.mjs" 446 ); 447 } 448 449 const gInterestingCategories = new Set([ 450 "addon-provider-module", 451 "webextension-modules", 452 "webextension-scripts", 453 "webextension-schemas", 454 "webextension-scripts-addon", 455 "webextension-scripts-content", 456 "webextension-scripts-devtools", 457 ]); 458 459 var gChromeReg = Cc["@mozilla.org/chrome/chrome-registry;1"].getService( 460 Ci.nsIChromeRegistry 461 ); 462 var gChromeMap = new Map(); 463 var gOverrideMap = new Map(); 464 465 // In this map when the value is a Set of URLs, the file is referenced if any 466 // of the files in the Set is referenced. 467 // When the value is null, the file is referenced unconditionally. 468 // When the value is a string, "allowlist-direct" means that we have not found 469 // any reference in the code, but have a matching allowlist entry for this file. 470 // "allowlist" means that the file is indirectly allowlisted, ie. a allowlisted 471 // file causes this file to be referenced. 472 var gReferencesFromCode = new Map(); 473 474 var resHandler = Services.io 475 .getProtocolHandler("resource") 476 .QueryInterface(Ci.nsIResProtocolHandler); 477 var gResourceMap = []; 478 function trackResourcePrefix(prefix) { 479 let uri = Services.io.newURI("resource://" + prefix + "/"); 480 gResourceMap.unshift([prefix, resHandler.resolveURI(uri)]); 481 } 482 trackResourcePrefix("gre"); 483 trackResourcePrefix("app"); 484 485 function getBaseUriForChromeUri(chromeUri) { 486 let chromeFile = chromeUri + "nonexistentfile.reallynothere"; 487 let uri = Services.io.newURI(chromeFile); 488 let fileUri = gChromeReg.convertChromeURL(uri); 489 return fileUri.resolve("."); 490 } 491 492 function trackChromeUri(uri) { 493 gChromeMap.set(getBaseUriForChromeUri(uri), uri); 494 } 495 496 // formautofill registers resource://formautofill/ and 497 // chrome://formautofill/content/ dynamically at runtime. 498 // Bug 1480276 is about addressing this without this hard-coding. 499 trackResourcePrefix("autofill"); 500 trackChromeUri("chrome://formautofill/content/"); 501 502 function parseManifest(manifestUri) { 503 return fetchFile(manifestUri.spec).then(data => { 504 for (let line of data.split("\n")) { 505 let [type, ...argv] = line.split(/\s+/); 506 if (type == "content" || type == "skin" || type == "locale") { 507 let chromeUri = `chrome://${argv[0]}/${type}/`; 508 trackChromeUri(chromeUri); 509 } else if (type == "override" || type == "overlay") { 510 // Overlays aren't really overrides, but behave the same in 511 // that the overlay is only referenced if the original xul 512 // file is referenced somewhere. 513 let os = "os=" + Services.appinfo.OS; 514 if (!argv.some(s => s.startsWith("os=") && s != os)) { 515 gOverrideMap.set( 516 Services.io.newURI(argv[1]).specIgnoringRef, 517 Services.io.newURI(argv[0]).specIgnoringRef 518 ); 519 } 520 } else if (type == "category") { 521 if (gInterestingCategories.has(argv[0])) { 522 gReferencesFromCode.set(argv[2], null); 523 } else if ( 524 argv[1].startsWith("resource://") || 525 argv[1].startsWith("moz-src://") 526 ) { 527 // Assume that any resource paths immediately after the category name 528 // are for use with BrowserUtils.callModulesFromCategory (rather than 529 // having to hardcode a list of categories in this test). 530 gReferencesFromCode.set(argv[1], null); 531 } 532 } else if (type == "resource") { 533 trackResourcePrefix(argv[0]); 534 } 535 } 536 }); 537 } 538 539 // If the given URI is a webextension manifest, extract files used by 540 // any of its APIs (scripts, icons, style sheets, theme images). 541 // Returns the passed in URI if the manifest is not a webextension 542 // manifest, null otherwise. 543 async function parseJsonManifest(uri) { 544 uri = Services.io.newURI(convertToCodeURI(uri.spec)); 545 546 let raw = await fetchFile(uri.spec); 547 let data; 548 try { 549 data = JSON.parse(raw); 550 } catch (ex) { 551 return uri; 552 } 553 554 // Simplistic test for whether this is a webextension manifest: 555 if (data.manifest_version !== 2) { 556 return uri; 557 } 558 559 if (data.background?.scripts) { 560 for (let bgscript of data.background.scripts) { 561 gReferencesFromCode.set(uri.resolve(bgscript), null); 562 } 563 } 564 565 if (data.icons) { 566 for (let icon of Object.values(data.icons)) { 567 gReferencesFromCode.set(uri.resolve(icon), null); 568 } 569 } 570 571 if (data.experiment_apis) { 572 for (let api of Object.values(data.experiment_apis)) { 573 if (api.parent && api.parent.script) { 574 let script = uri.resolve(api.parent.script); 575 gReferencesFromCode.set(script, null); 576 } 577 578 if (api.schema) { 579 gReferencesFromCode.set(uri.resolve(api.schema), null); 580 } 581 } 582 } 583 584 if (data.theme_experiment && data.theme_experiment.stylesheet) { 585 let stylesheet = uri.resolve(data.theme_experiment.stylesheet); 586 gReferencesFromCode.set(stylesheet, null); 587 } 588 589 for (let themeKey of ["theme", "dark_theme"]) { 590 if (data?.[themeKey]?.images?.additional_backgrounds) { 591 for (let background of data[themeKey].images.additional_backgrounds) { 592 gReferencesFromCode.set(uri.resolve(background), null); 593 } 594 } 595 } 596 597 return null; 598 } 599 600 function addCodeReference(url, fromURI) { 601 let from = convertToCodeURI(fromURI.spec); 602 603 // Ignore self references. 604 if (url == from) { 605 return; 606 } 607 608 let ref; 609 if (gReferencesFromCode.has(url)) { 610 ref = gReferencesFromCode.get(url); 611 if (ref === null) { 612 return; 613 } 614 } else { 615 ref = new Set(); 616 gReferencesFromCode.set(url, ref); 617 } 618 ref.add(from); 619 } 620 621 function listCodeReferences(refs) { 622 let refList = []; 623 if (refs) { 624 for (let ref of refs) { 625 refList.push(ref); 626 } 627 } 628 return refList.join(","); 629 } 630 631 function parseCSSFile(fileUri) { 632 return fetchFile(fileUri.spec).then(data => { 633 for (let line of data.split("\n")) { 634 let urls = line.match(/url\([^()]+\)/g); 635 if (!urls) { 636 // @import rules can take a string instead of a url. 637 let importMatch = line.match(/@import ['"]?([^'"]*)['"]?/); 638 if (importMatch && importMatch[1]) { 639 let url = Services.io.newURI(importMatch[1], null, fileUri).spec; 640 addCodeReference(convertToCodeURI(url), fileUri); 641 } 642 continue; 643 } 644 645 for (let url of urls) { 646 // Remove the url(" prefix and the ") suffix. 647 url = url 648 .replace(/url\(([^)]*)\)/, "$1") 649 .replace(/^"(.*)"$/, "$1") 650 .replace(/^'(.*)'$/, "$1"); 651 if (url.startsWith("data:")) { 652 continue; 653 } 654 655 try { 656 url = Services.io.newURI(url, null, fileUri).specIgnoringRef; 657 addCodeReference(convertToCodeURI(url), fileUri); 658 } catch (e) { 659 ok(false, "unexpected error while resolving this URI: " + url); 660 } 661 } 662 } 663 }); 664 } 665 666 function parseCodeFile(fileUri) { 667 return fetchFile(fileUri.spec).then(data => { 668 let baseUri; 669 for (let line of data.split("\n")) { 670 let urls = line.match( 671 /["'`]chrome:\/\/[a-zA-Z0-9-]+\/(content|skin|locale)\/[^"'` ]*["'`]/g 672 ); 673 674 if (!urls) { 675 urls = line.match(/["']moz-src:\/\/[^"']+["']/g); 676 } 677 678 if (!urls) { 679 urls = line.match(/["']resource:\/\/[^"']+["']/g); 680 if ( 681 urls && 682 isDevtools && 683 /baseURI: "resource:\/\/devtools\//.test(line) 684 ) { 685 baseUri = Services.io.newURI(urls[0].slice(1, -1)); 686 continue; 687 } 688 } 689 690 if (!urls) { 691 urls = line.match(/[a-z0-9_\/-]+\.ftl/i); 692 if (urls) { 693 urls = urls[0]; 694 let grePrefix = Services.io.newURI( 695 "resource://gre/localization/en-US/" 696 ); 697 let appPrefix = Services.io.newURI( 698 "resource://app/localization/en-US/" 699 ); 700 701 let grePrefixUrl = Services.io.newURI(urls, null, grePrefix).spec; 702 let appPrefixUrl = Services.io.newURI(urls, null, appPrefix).spec; 703 704 addCodeReference(grePrefixUrl, fileUri); 705 addCodeReference(appPrefixUrl, fileUri); 706 continue; 707 } 708 } 709 710 if (!urls) { 711 // If there's no absolute chrome URL, look for relative ones in 712 // src and href attributes. 713 let match = line.match("(?:src|href)=[\"']([^$&\"']+)"); 714 if (match && match[1]) { 715 let url = Services.io.newURI(match[1], null, fileUri).spec; 716 addCodeReference(convertToCodeURI(url), fileUri); 717 } 718 719 // This handles `import` lines which may be multi-line. 720 // We have an ESLint rule, `import/no-unassigned-import` which prevents 721 // using bare `import "foo.js"`, so we don't need to handle that case 722 // here. 723 match = line.match(/from\W*['"](.*?)['"]/); 724 if (match?.[1]) { 725 let url = match[1]; 726 url = Services.io.newURI(url, null, baseUri || fileUri).spec; 727 url = convertToCodeURI(url); 728 addCodeReference(url, fileUri); 729 } 730 731 if (isDevtools) { 732 let rules = [ 733 ["devtools/client/locales", "chrome://devtools/locale"], 734 ["devtools/shared/locales", "chrome://devtools-shared/locale"], 735 [ 736 "devtools/shared/platform", 737 "resource://devtools/shared/platform/chrome", 738 ], 739 ["devtools", "resource://devtools"], 740 ]; 741 742 match = line.match(/["']((?:devtools)\/[^\\#"']+)["']/); 743 if (match && match[1]) { 744 let path = match[1]; 745 for (let rule of rules) { 746 if (path.startsWith(rule[0] + "/")) { 747 path = path.replace(rule[0], rule[1]); 748 if (!/\.(properties|js|jsm|mjs|json|css)$/.test(path)) { 749 path += ".js"; 750 } 751 addCodeReference(path, fileUri); 752 break; 753 } 754 } 755 } 756 757 match = line.match(/require\(['"](\.[^'"]+)['"]\)/); 758 if (match && match[1]) { 759 let url = match[1]; 760 url = Services.io.newURI(url, null, baseUri || fileUri).spec; 761 url = convertToCodeURI(url); 762 if (!/\.(properties|js|jsm|mjs|json|css)$/.test(url)) { 763 url += ".js"; 764 } 765 if ( 766 url.startsWith("resource://") || 767 url.startsWith("moz-src:///") 768 ) { 769 addCodeReference(url, fileUri); 770 } else { 771 // if we end up with a chrome:// url here, it's likely because 772 // a baseURI to a resource:// path has been defined in another 773 // .js file that is loaded in the same scope, we can't detect it. 774 } 775 } 776 } 777 continue; 778 } 779 780 for (let url of urls) { 781 // Remove quotes. 782 url = url.slice(1, -1); 783 // Remove ? or \ trailing characters. 784 if (url.endsWith("\\")) { 785 url = url.slice(0, -1); 786 } 787 788 let pos = url.indexOf("?"); 789 if (pos != -1) { 790 url = url.slice(0, pos); 791 } 792 793 // Make urls like chrome://browser/skin/ point to an actual file, 794 // and remove the ref if any. 795 try { 796 url = Services.io.newURI(url).specIgnoringRef; 797 } catch (e) { 798 continue; 799 } 800 801 if ( 802 isDevtools && 803 line.includes("require(") && 804 !/\.(properties|js|jsm|mjs|json|css)$/.test(url) 805 ) { 806 url += ".js"; 807 } 808 809 addCodeReference(url, fileUri); 810 } 811 } 812 }); 813 } 814 815 function convertToCodeURI(fileUri) { 816 let baseUri = fileUri; 817 let path = ""; 818 while (baseUri) { 819 let slashPos = baseUri.lastIndexOf("/", baseUri.length - 2); 820 if (slashPos <= 0) { 821 // File not accessible from chrome protocol, try resource:// 822 for (let res of gResourceMap) { 823 if (fileUri.startsWith(res[1])) { 824 let resourceUriString = fileUri.replace( 825 res[1], 826 `resource://${res[0]}/` 827 ); 828 // If inside moz-src, treat as moz-src url. 829 resourceUriString = resourceUriString.replace( 830 /^resource:\/\/gre\/moz-src\//, 831 "moz-src:///" 832 ); 833 return resourceUriString; 834 } 835 } 836 // Give up and return the original URL. 837 return fileUri; 838 } 839 path = baseUri.slice(slashPos + 1) + path; 840 baseUri = baseUri.slice(0, slashPos + 1); 841 if (gChromeMap.has(baseUri)) { 842 return gChromeMap.get(baseUri) + path; 843 } 844 } 845 throw new Error(`Unparsable URI: ${fileUri}`); 846 } 847 848 async function chromeFileExists(aURI) { 849 try { 850 return await PerfTestHelpers.checkURIExists(aURI); 851 } catch (e) { 852 todo(false, `Failed to check if ${aURI} exists: ${e}`); 853 return false; 854 } 855 } 856 857 function findChromeUrlsFromArray(array, prefix) { 858 // Find the first character of the prefix... 859 for ( 860 let index = 0; 861 (index = array.indexOf(prefix.charCodeAt(0), index)) != -1; 862 ++index 863 ) { 864 // Then ensure we actually have the whole prefix. 865 let found = true; 866 for (let i = 1; i < prefix.length; ++i) { 867 if (array[index + i] != prefix.charCodeAt(i)) { 868 found = false; 869 break; 870 } 871 } 872 if (!found) { 873 continue; 874 } 875 876 // C strings are null terminated, but " also terminates urls 877 // (nsIndexedToHTML.cpp contains an HTML fragment with several chrome urls) 878 // Let's also terminate the string on the # character to skip references. 879 let end = Math.min( 880 array.indexOf(0, index), 881 array.indexOf('"'.charCodeAt(0), index), 882 array.indexOf(")".charCodeAt(0), index), 883 array.indexOf("#".charCodeAt(0), index) 884 ); 885 let string = ""; 886 for (; index < end; ++index) { 887 string += String.fromCharCode(array[index]); 888 } 889 890 // Only keep strings that look like real chrome or resource urls. 891 if ( 892 /chrome:\/\/[a-zA-Z09-]+\/(content|skin|locale)\//.test(string) || 893 /moz-src:\/\/\/\w+/.test(string) || 894 /resource:\/\/[a-zA-Z09-]*\/.*\.[a-z]+/.test(string) 895 ) { 896 gReferencesFromCode.set(string, null); 897 } 898 } 899 } 900 901 add_task(async function checkAllTheFiles() { 902 TestUtils.assertPackagedBuild(); 903 904 const libxul = await IOUtils.read(PathUtils.xulLibraryPath); 905 findChromeUrlsFromArray(libxul, "chrome://"); 906 findChromeUrlsFromArray(libxul, "resource://"); 907 findChromeUrlsFromArray(libxul, "moz-src://"); 908 // Handle NS_LITERAL_STRING. 909 let uint16 = new Uint16Array(libxul.buffer); 910 findChromeUrlsFromArray(uint16, "chrome://"); 911 findChromeUrlsFromArray(uint16, "resource://"); 912 findChromeUrlsFromArray(uint16, "moz-src://"); 913 914 const kCodeExtensions = [ 915 ".xml", 916 ".xsl", 917 ".mjs", 918 ".js", 919 ".json", 920 ".html", 921 ".xhtml", 922 ]; 923 924 let appDir = Services.dirsvc.get("GreD", Ci.nsIFile); 925 // This asynchronously produces a list of URLs (sadly, mostly sync on our 926 // test infrastructure because it runs against jarfiles there, and 927 // our zipreader APIs are all sync) 928 let uris = await generateURIsFromDirTree( 929 appDir, 930 [ 931 ".css", 932 ".manifest", 933 ".jpg", 934 ".png", 935 ".gif", 936 ".svg", 937 ".ftl", 938 ".dtd", 939 ".properties", 940 ].concat(kCodeExtensions) 941 ); 942 943 // Parse and remove all manifests from the list. 944 // NOTE that this must be done before filtering out devtools paths 945 // so that all chrome paths can be recorded. 946 let manifestURIs = []; 947 let jsonManifests = []; 948 uris = uris.filter(uri => { 949 let path = uri.pathQueryRef; 950 if (path.endsWith(".manifest")) { 951 manifestURIs.push(uri); 952 return false; 953 } else if (path.endsWith("/manifest.json")) { 954 jsonManifests.push(uri); 955 return false; 956 } 957 958 return true; 959 }); 960 961 // Wait for all manifest to be parsed 962 await PerfTestHelpers.throttledMapPromises(manifestURIs, parseManifest); 963 964 for (let esModule of Components.manager.getComponentESModules()) { 965 gReferencesFromCode.set(esModule, null); 966 } 967 968 // manifest.json is a common name, it is used for WebExtension manifests 969 // but also for other things. To tell them apart, we have to actually 970 // read the contents. This will populate gExtensionRoots with all 971 // embedded extension APIs, and return any manifest.json files that aren't 972 // webextensions. 973 let nonWebextManifests = ( 974 await Promise.all(jsonManifests.map(parseJsonManifest)) 975 ).filter(uri => !!uri); 976 uris.push(...nonWebextManifests); 977 978 // We build a list of promises that get resolved when their respective 979 // files have loaded and produced no errors. 980 let allPromises = []; 981 982 for (let uri of uris) { 983 let path = uri.pathQueryRef; 984 if (path.endsWith(".css")) { 985 allPromises.push([parseCSSFile, uri]); 986 } else if (kCodeExtensions.some(ext => path.endsWith(ext))) { 987 allPromises.push([parseCodeFile, uri]); 988 } 989 } 990 991 // Wait for all the files to have actually loaded: 992 await PerfTestHelpers.throttledMapPromises(allPromises, ([task, uri]) => 993 task(uri) 994 ); 995 996 // Keep only chrome:// files, and filter out either the devtools paths or 997 // the non-devtools paths: 998 let devtoolsPrefixes = [ 999 "chrome://devtools", 1000 "moz-src:///devtools/", 1001 "resource://devtools/", 1002 "resource://devtools-shared-images/", 1003 "resource://devtools-highlighter-styles/", 1004 "resource://app/modules/devtools", 1005 "resource://gre/modules/devtools", 1006 "resource://app/localization/en-US/startup/aboutDevTools.ftl", 1007 "resource://app/localization/en-US/devtools/", 1008 ]; 1009 let hasDevtoolsPrefix = uri => 1010 devtoolsPrefixes.some(prefix => uri.startsWith(prefix)); 1011 let chromeFiles = []; 1012 for (let uri of uris) { 1013 uri = convertToCodeURI(uri.spec); 1014 if ( 1015 (uri.startsWith("chrome://") || 1016 uri.startsWith("resource://") || 1017 uri.startsWith("moz-src:///")) && 1018 isDevtools == hasDevtoolsPrefix(uri) 1019 ) { 1020 chromeFiles.push(uri); 1021 } 1022 } 1023 1024 if (isDevtools) { 1025 // chrome://devtools/skin/devtools-browser.css is included from browser.xhtml 1026 gReferencesFromCode.set(AppConstants.BROWSER_CHROME_URL, null); 1027 // devtools' css is currently included from browser.css, see bug 1204810. 1028 gReferencesFromCode.set("chrome://browser/skin/browser.css", null); 1029 } 1030 1031 let isUnreferenced = file => { 1032 if (gExceptionPaths.some(e => file.startsWith(e))) { 1033 return false; 1034 } 1035 if (gReferencesFromCode.has(file)) { 1036 let refs = gReferencesFromCode.get(file); 1037 if (refs === null) { 1038 return false; 1039 } 1040 for (let ref of refs) { 1041 if (isDevtools) { 1042 if ( 1043 ref.startsWith("resource://app/components/") || 1044 (file.startsWith("chrome://") && 1045 (ref.startsWith("resource://") || ref.startsWith("moz-src://"))) 1046 ) { 1047 return false; 1048 } 1049 } 1050 1051 if (gReferencesFromCode.has(ref)) { 1052 let refType = gReferencesFromCode.get(ref); 1053 if ( 1054 refType === null || // unconditionally referenced 1055 refType == "allowlist" || 1056 refType == "allowlist-direct" 1057 ) { 1058 return false; 1059 } 1060 } 1061 } 1062 } 1063 return !gOverrideMap.has(file) || isUnreferenced(gOverrideMap.get(file)); 1064 }; 1065 1066 let unreferencedFiles = chromeFiles; 1067 1068 let removeReferenced = useAllowlist => { 1069 let foundReference = false; 1070 unreferencedFiles = unreferencedFiles.filter(f => { 1071 let rv = isUnreferenced(f); 1072 if (rv && f.startsWith("resource://app/")) { 1073 rv = isUnreferenced(f.replace("resource://app/", "resource:///")); 1074 } 1075 if (!rv) { 1076 foundReference = true; 1077 if (useAllowlist) { 1078 info( 1079 "indirectly allowlisted file: " + 1080 f + 1081 " used from " + 1082 listCodeReferences(gReferencesFromCode.get(f)) 1083 ); 1084 } 1085 gReferencesFromCode.set(f, useAllowlist ? "allowlist" : null); 1086 } 1087 return rv; 1088 }); 1089 return foundReference; 1090 }; 1091 // First filter out the files that are referenced. 1092 while (removeReferenced(false)) { 1093 // As long as removeReferenced returns true, some files have been marked 1094 // as referenced, so we need to run it again. 1095 } 1096 // Marked as referenced the files that have been explicitly allowed. 1097 unreferencedFiles = unreferencedFiles.filter(file => { 1098 if (allowlist.has(file)) { 1099 allowlist.delete(file); 1100 gReferencesFromCode.set(file, "allowlist-direct"); 1101 return false; 1102 } 1103 return true; 1104 }); 1105 // Run the process again, this time when more files are marked as referenced, 1106 // it's a consequence of the allowlist. 1107 while (removeReferenced(true)) { 1108 // As long as removeReferenced returns true, we need to run it again. 1109 } 1110 1111 unreferencedFiles.sort(); 1112 1113 if (isDevtools) { 1114 // Bug 1351878 - handle devtools resource files 1115 unreferencedFiles = unreferencedFiles.filter(file => { 1116 if (file.startsWith("resource://")) { 1117 info("unreferenced devtools resource file: " + file); 1118 return false; 1119 } 1120 return true; 1121 }); 1122 } 1123 1124 is(unreferencedFiles.length, 0, "there should be no unreferenced files"); 1125 for (let file of unreferencedFiles) { 1126 let refs = gReferencesFromCode.get(file); 1127 if (refs === undefined) { 1128 ok(false, "unreferenced file: " + file); 1129 } else { 1130 let refList = listCodeReferences(refs); 1131 let msg = "file only referenced from unreferenced files: " + file; 1132 if (refList) { 1133 msg += " referenced from " + refList; 1134 } 1135 ok(false, msg); 1136 } 1137 } 1138 1139 for (let file of allowlist) { 1140 if (ignorableAllowlist.has(file)) { 1141 info("ignored unused allowlist entry: " + file); 1142 } else { 1143 ok(false, "unused allowlist entry: " + file); 1144 } 1145 } 1146 1147 for (let [file, refs] of gReferencesFromCode) { 1148 if ( 1149 isDevtools != devtoolsPrefixes.some(prefix => file.startsWith(prefix)) 1150 ) { 1151 continue; 1152 } 1153 1154 if ( 1155 (file.startsWith("chrome://") || 1156 file.startsWith("resource://") || 1157 file.startsWith("moz-src:///")) && 1158 !(await chromeFileExists(file)) 1159 ) { 1160 // Ignore chrome prefixes that have been automatically expanded. 1161 let pathParts = 1162 file.match("chrome://([^/]+)/content/([^/.]+).xul") || 1163 file.match("chrome://([^/]+)/skin/([^/.]+).css"); 1164 if (pathParts && pathParts[1] == pathParts[2]) { 1165 continue; 1166 } 1167 1168 // TODO: bug 1349010 - add a allowlist and make this reliable enough 1169 // that we could make the test fail when this catches something new. 1170 let refList = listCodeReferences(refs); 1171 let msg = "missing file: " + file; 1172 if (refList) { 1173 msg += " referenced from " + refList; 1174 } 1175 info(msg); 1176 } 1177 } 1178 });