test_quicksuggest_relevanceRanking.js (11689B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 // Tests relevance ranking integration with UrlbarProviderQuickSuggest. 6 7 "use strict"; 8 9 const lazy = {}; 10 11 ChromeUtils.defineESModuleGetters(lazy, { 12 ContentRelevancyManager: 13 "resource://gre/modules/ContentRelevancyManager.sys.mjs", 14 InterestVector: 15 "moz-src:///toolkit/components/uniffi-bindgen-gecko-js/components/generated/RustRelevancy.sys.mjs", 16 }); 17 18 const PREF_CONTENT_RELEVANCY_ENABLED = "toolkit.contentRelevancy.enabled"; 19 const PREF_RANKING_MODE = "browser.urlbar.quicksuggest.rankingMode"; 20 21 function makeTestSuggestions() { 22 return [ 23 { 24 title: "suggestion_about_education", 25 categories: [6], // "Education" 26 score: 0.2, 27 }, 28 { 29 title: "suggestion_about_animals", 30 categories: [1], // "Animals" 31 score: 0.2, 32 }, 33 ]; 34 } 35 36 function makeTestSuggestionsWithInvalidCategories() { 37 return [ 38 { 39 title: "suggestion", 40 categories: [-1], // "Education" 41 score: 0.2, 42 }, 43 ]; 44 } 45 46 const MERINO_SUGGESTIONS = [ 47 { 48 provider: "adm", 49 full_keyword: "amp", 50 title: "Amp Suggestion", 51 url: "https://example.com/amp", 52 icon: null, 53 impression_url: "https://example.com/amp-impression", 54 click_url: "https://example.com/amp-click", 55 block_id: 1, 56 advertiser: "Amp", 57 iab_category: "22 - Shopping", 58 is_sponsored: true, 59 categories: [1], // Animals 60 score: 0.3, 61 }, 62 { 63 title: "Wikipedia Suggestion", 64 url: "https://example.com/wikipedia", 65 provider: "wikipedia", 66 full_keyword: "wikipedia", 67 icon: null, 68 block_id: 0, 69 advertiser: "dynamic-Wikipedia", 70 is_sponsored: false, 71 categories: [6], // Education 72 score: 0.23, 73 }, 74 ]; 75 76 const SEARCH_STRING = "frab"; 77 78 const EXPECTED_AMP_RESULT = QuickSuggestTestUtils.ampResult({ 79 source: "merino", 80 provider: "adm", 81 requestId: "request_id", 82 suggestedIndex: -1, 83 }); 84 const EXPECTED_WIKIPEDIA_RESULT = QuickSuggestTestUtils.wikipediaResult({ 85 source: "merino", 86 provider: "wikipedia", 87 telemetryType: "wikipedia", 88 }); 89 90 let gSandbox; 91 92 add_setup(async () => { 93 // FOG needs a profile directory to put its data in. 94 do_get_profile(); 95 96 // FOG needs to be initialized in order for data to flow. 97 Services.fog.initializeFOG(); 98 99 await QuickSuggestTestUtils.ensureQuickSuggestInit({ 100 merinoSuggestions: MERINO_SUGGESTIONS, 101 prefs: [ 102 ["suggest.quicksuggest.all", true], 103 ["suggest.quicksuggest.sponsored", true], 104 105 // Turn off higher-placement sponsored so this test doesn't need to worry 106 // about best matches. 107 ["quicksuggest.ampTopPickCharThreshold", 0], 108 ], 109 }); 110 gSandbox = sinon.createSandbox(); 111 112 const fakeStore = { 113 close: gSandbox.fake(), 114 userInterestVector: gSandbox.stub(), 115 }; 116 const rustRelevancyStore = { 117 init: gSandbox.fake.returns(fakeStore), 118 }; 119 fakeStore.userInterestVector.resolves( 120 new lazy.InterestVector({ 121 animals: 0, 122 arts: 0, 123 autos: 0, 124 business: 0, 125 career: 0, 126 education: 50, 127 fashion: 0, 128 finance: 0, 129 food: 0, 130 government: 0, 131 hobbies: 0, 132 home: 0, 133 news: 0, 134 realEstate: 0, 135 society: 0, 136 sports: 0, 137 tech: 0, 138 travel: 0, 139 inconclusive: 0, 140 }) 141 ); 142 143 Services.prefs.setBoolPref(PREF_CONTENT_RELEVANCY_ENABLED, true); 144 lazy.ContentRelevancyManager.init(rustRelevancyStore); 145 146 registerCleanupFunction(() => { 147 lazy.ContentRelevancyManager.uninit(); 148 Services.prefs.clearUserPref(PREF_CONTENT_RELEVANCY_ENABLED); 149 gSandbox.restore(); 150 }); 151 }); 152 153 add_task(async function test_interest_mode() { 154 Services.prefs.setStringPref(PREF_RANKING_MODE, "interest"); 155 156 const suggestions = makeTestSuggestions(); 157 await applyRanking(suggestions); 158 159 Assert.greater( 160 suggestions[0].score, 161 0.2, 162 "The score should be boosted for relevant suggestions" 163 ); 164 Assert.less( 165 suggestions[1].score, 166 0.2, 167 "The score should be lowered for irrelevant suggestion" 168 ); 169 170 Services.prefs.clearUserPref(PREF_RANKING_MODE); 171 }); 172 173 add_task(async function test_default_mode() { 174 Services.prefs.setStringPref(PREF_RANKING_MODE, "default"); 175 176 const suggestions = makeTestSuggestions(); 177 await applyRanking(suggestions); 178 179 Assert.equal( 180 suggestions[0].score, 181 0.2, 182 "The score should be unchanged for the default mode" 183 ); 184 Assert.equal( 185 suggestions[1].score, 186 0.2, 187 "The score should be unchanged for the default mode" 188 ); 189 190 Services.prefs.clearUserPref(PREF_RANKING_MODE); 191 }); 192 193 add_task(async function test_random_mode() { 194 Services.prefs.setStringPref(PREF_RANKING_MODE, "random"); 195 196 const suggestions = makeTestSuggestions(); 197 await applyRanking(suggestions); 198 199 for (let s of suggestions) { 200 Assert.equal(typeof s.score, "number", "Suggestion should have a score"); 201 Assert.greaterOrEqual(s.score, 0, "Suggestion score should be >= 0"); 202 Assert.lessOrEqual(s.score, 1, "Suggestion score should be <= 1"); 203 Assert.notEqual( 204 s.score, 205 0.2, 206 "Suggestion score should be different from its initial value (probably!)" 207 ); 208 } 209 210 let uniqueScores = new Set(suggestions.map(s => s.score)); 211 Assert.equal( 212 uniqueScores.size, 213 suggestions.length, 214 "Suggestion scores should be unique (probably!)" 215 ); 216 217 Services.prefs.clearUserPref(PREF_RANKING_MODE); 218 }); 219 220 add_task(async function test_default_mode_end2end() { 221 Services.prefs.setStringPref(PREF_RANKING_MODE, "default"); 222 223 let context = createContext(SEARCH_STRING, { 224 providers: [UrlbarProviderQuickSuggest.name], 225 isPrivate: false, 226 }); 227 228 await check_results({ 229 context, 230 matches: [EXPECTED_AMP_RESULT], 231 }); 232 233 Services.prefs.clearUserPref(PREF_RANKING_MODE); 234 }); 235 236 add_task(async function test_interest_mode_end2end() { 237 Services.prefs.setStringPref(PREF_RANKING_MODE, "interest"); 238 239 let context = createContext(SEARCH_STRING, { 240 providers: [UrlbarProviderQuickSuggest.name], 241 isPrivate: false, 242 }); 243 244 await check_results({ 245 context, 246 matches: [EXPECTED_WIKIPEDIA_RESULT], 247 }); 248 249 Services.prefs.clearUserPref(PREF_RANKING_MODE); 250 }); 251 252 add_task(async function test_telemetry_interest_mode() { 253 Services.prefs.setStringPref(PREF_RANKING_MODE, "interest"); 254 255 Services.fog.testResetFOG(); 256 257 Assert.equal(null, Glean.suggestRelevance.status.success.testGetValue()); 258 Assert.equal(null, Glean.suggestRelevance.status.failure.testGetValue()); 259 Assert.equal(null, Glean.suggestRelevance.outcome.boosted.testGetValue()); 260 Assert.equal(null, Glean.suggestRelevance.outcome.decreased.testGetValue()); 261 262 const suggestions = makeTestSuggestions(); 263 await applyRanking(suggestions); 264 265 // The scoring should succeed for both suggestions with one boosted score 266 // and one decreased score. 267 Assert.equal(2, Glean.suggestRelevance.status.success.testGetValue()); 268 Assert.equal(null, Glean.suggestRelevance.status.failure.testGetValue()); 269 Assert.equal(1, Glean.suggestRelevance.outcome.boosted.testGetValue()); 270 Assert.equal(1, Glean.suggestRelevance.outcome.decreased.testGetValue()); 271 272 Services.prefs.clearUserPref(PREF_RANKING_MODE); 273 }); 274 275 add_task(async function test_telemetry_interest_mode_with_failures() { 276 Services.prefs.setStringPref(PREF_RANKING_MODE, "interest"); 277 278 Services.fog.testResetFOG(); 279 280 Assert.equal(null, Glean.suggestRelevance.status.success.testGetValue()); 281 Assert.equal(null, Glean.suggestRelevance.status.failure.testGetValue()); 282 Assert.equal(null, Glean.suggestRelevance.outcome.boosted.testGetValue()); 283 Assert.equal(null, Glean.suggestRelevance.outcome.decreased.testGetValue()); 284 285 const suggestions = makeTestSuggestionsWithInvalidCategories(); 286 await applyRanking(suggestions); 287 288 // The scoring should fail. 289 Assert.equal(null, Glean.suggestRelevance.status.success.testGetValue()); 290 Assert.equal(1, Glean.suggestRelevance.status.failure.testGetValue()); 291 Assert.equal(null, Glean.suggestRelevance.outcome.boosted.testGetValue()); 292 Assert.equal(null, Glean.suggestRelevance.outcome.decreased.testGetValue()); 293 294 Services.prefs.clearUserPref(PREF_RANKING_MODE); 295 }); 296 297 add_task(async function offline_interest_mode_end2end() { 298 // Interest mode should return the suggestion whose category has the largest 299 // interest vector value: the Education suggestion. 300 await doOfflineTest({ 301 mode: "interest", 302 expectedResultArgs: { 303 url: "https://example.com/6-education", 304 title: "Suggestion with category 6 (Education)", 305 }, 306 }); 307 }); 308 309 add_task(async function offline_default_mode_end2end() { 310 // Default mode should return the first suggestion with the highest score, 311 // which is just the first suggestion returned by the backend since they all 312 // have the same score. 313 await doOfflineTest({ 314 mode: "default", 315 expectedResultArgs: { 316 url: "https://example.com/no-categories", 317 title: "Suggestion with no categories", 318 }, 319 }); 320 }); 321 322 async function doOfflineTest({ mode, expectedResultArgs }) { 323 // Turn off Merino. 324 UrlbarPrefs.set("quicksuggest.online.enabled", false); 325 326 Services.prefs.setStringPref(PREF_RANKING_MODE, mode); 327 328 // TODO: For now we stub `query()` on the Rust backend so that it returns AMP 329 // suggestions that have the `categories` property. Once the Rust component 330 // actually returns AMP suggestions with `categories`, we should be able to 331 // remove this and instead pass appropriate AMP data in the setup task to 332 // `QuickSuggestTestUtils.ensureQuickSuggestInit()`. When we do, make sure all 333 // suggestions have the same keyword! We want Rust to return all of them in 334 // response to a single query so that they are sorted and chosen by relevancy 335 // ranking. 336 let sandbox = sinon.createSandbox(); 337 let queryStub = sandbox.stub(QuickSuggest.rustBackend, "query"); 338 queryStub.returns([ 339 mockRustAmpSuggestion({ 340 keyword: "offline", 341 title: "Suggestion with no categories", 342 url: "https://example.com/no-categories", 343 categories: [], 344 }), 345 mockRustAmpSuggestion({ 346 keyword: "offline", 347 url: "https://example.com/6-education", 348 title: "Suggestion with category 6 (Education)", 349 categories: [6], 350 }), 351 mockRustAmpSuggestion({ 352 keyword: "offline", 353 title: "Suggestion with category 1 (Animals)", 354 url: "https://example.com/1-animals", 355 categories: [1], 356 }), 357 ]); 358 359 await check_results({ 360 context: createContext("offline", { 361 providers: [UrlbarProviderQuickSuggest.name], 362 isPrivate: false, 363 }), 364 matches: [ 365 QuickSuggestTestUtils.ampResult({ 366 ...expectedResultArgs, 367 keyword: "offline", 368 suggestedIndex: -1, 369 }), 370 ], 371 }); 372 373 Services.prefs.clearUserPref(PREF_RANKING_MODE); 374 UrlbarPrefs.clear("quicksuggest.online.enabled"); 375 sandbox.restore(); 376 } 377 378 async function applyRanking(suggestions) { 379 let quickSuggestProviderInstance = UrlbarProvidersManager.getProvider( 380 UrlbarProviderQuickSuggest.name 381 ); 382 for (let s of suggestions) { 383 await quickSuggestProviderInstance._test_applyRanking(s); 384 } 385 } 386 387 function mockRustAmpSuggestion({ keyword, url, title, categories }) { 388 let suggestion = QuickSuggestTestUtils.ampRemoteSettings({ 389 url, 390 title, 391 keywords: [keyword], 392 }); 393 return { 394 ...suggestion, 395 rawUrl: suggestion.url, 396 impressionUrl: suggestion.impression_url, 397 clickUrl: suggestion.click_url, 398 blockId: suggestion.id, 399 iabCategory: suggestion.iab_category, 400 icon: null, 401 fullKeyword: keyword, 402 source: "rust", 403 provider: "Amp", 404 categories, 405 }; 406 }