test_BackupService_retryHeuristic.js (7651B)
1 /* Any copyright is dedicated to the Public Domain. 2 https://creativecommons.org/publicdomain/zero/1.0/ */ 3 4 "use strict"; 5 6 ChromeUtils.defineESModuleGetters(this, { 7 BackupError: "resource:///modules/backup/BackupError.mjs", 8 ERRORS: "chrome://browser/content/backup/backup-constants.mjs", 9 TestUtils: "resource://testing-common/TestUtils.sys.mjs", 10 }); 11 12 const BACKUP_RETRY_LIMIT_PREF_NAME = "browser.backup.backup-retry-limit"; 13 const DISABLED_ON_IDLE_RETRY_PREF_NAME = 14 "browser.backup.disabled-on-idle-backup-retry"; 15 const BACKUP_ERROR_CODE_PREF_NAME = "browser.backup.errorCode"; 16 const MINIMUM_TIME_BETWEEN_BACKUPS_SECONDS_PREF_NAME = 17 "browser.backup.scheduled.minimum-time-between-backups-seconds"; 18 const SCHEDULED_BACKUPS_ENABLED_PREF_NAME = "browser.backup.scheduled.enabled"; 19 const BACKUP_DEBUG_INFO_PREF_NAME = "browser.backup.backup-debug-info"; 20 const BACKUP_DEFAULT_LOCATION_PREF_NAME = "browser.backup.location"; 21 22 const RETRIES_FOR_TEST = 4; 23 24 async function create_backup_failure_expected_calls( 25 bs, 26 callCount, 27 assertionMsg 28 ) { 29 assertionMsg = assertionMsg 30 ? assertionMsg 31 : `createBackup should be called ${callCount} times`; 32 33 let originalBackoffTime = BackupService.backoffSeconds(); 34 35 bs.createBackupOnIdleDispatch({}); 36 37 // testing that callCount remains the same, skip all the other checks 38 if (callCount == bs.createBackup.callCount) { 39 Assert.equal(bs.createBackup.callCount, callCount, assertionMsg); 40 41 return; 42 } 43 44 // Wait for in progress states to change 45 // so that the errorRetries can be updated 46 47 await bsInProgressStateUpdate(bs, true); 48 await bsInProgressStateUpdate(bs, false); 49 50 // propagate prefs 51 await TestUtils.waitForTick(); 52 53 // have we called createBackup more times than allowed retries? 54 // if so, the retries should reset and retrying should 55 // disable calling createBackup again 56 if (callCount == RETRIES_FOR_TEST + 1) { 57 Assert.equal( 58 Glean.browserBackup.backupThrottled.testGetValue().length, 59 1, 60 "backupThrottled telemetry was sent" 61 ); 62 63 Assert.ok( 64 Services.prefs.getBoolPref(DISABLED_ON_IDLE_RETRY_PREF_NAME), 65 "Disable on idle is now enabled - no more retries allowed" 66 ); 67 } 68 // we expect createBackup to be called, but it shouldn't succeed 69 else { 70 Assert.equal( 71 BackupService.backoffSeconds(), 72 2 * originalBackoffTime, 73 "Backoff time should have doubled" 74 ); 75 76 Assert.ok( 77 !Services.prefs.getBoolPref(DISABLED_ON_IDLE_RETRY_PREF_NAME), 78 "Disable on idle is disabled - which means that we can do more retries!" 79 ); 80 81 Assert.equal( 82 Glean.browserBackup.backupThrottled.testGetValue(), 83 null, 84 "backupThrottled telemetry was not sent yet" 85 ); 86 } 87 88 Assert.equal(bs.createBackup.callCount, callCount, assertionMsg); 89 90 Assert.equal( 91 Services.prefs.getIntPref(BACKUP_ERROR_CODE_PREF_NAME), 92 ERRORS.UNKNOWN, 93 "Error code has been set" 94 ); 95 } 96 97 function bsInProgressStateUpdate(bs, isBackupInProgress) { 98 // Check if already in desired state 99 if (bs.state.backupInProgress === isBackupInProgress) { 100 return Promise.resolve(); 101 } 102 103 return new Promise(resolve => { 104 const listener = () => { 105 if (bs.state.backupInProgress === isBackupInProgress) { 106 bs.removeEventListener("BackupService:StateUpdate", listener); 107 resolve(); 108 } 109 }; 110 111 bs.addEventListener("BackupService:StateUpdate", listener); 112 }); 113 } 114 115 add_setup(async () => { 116 const TEST_PROFILE_PATH = await IOUtils.createUniqueDirectory( 117 PathUtils.tempDir, 118 "testBackup" 119 ); 120 121 Services.prefs.setStringPref( 122 BACKUP_DEFAULT_LOCATION_PREF_NAME, 123 TEST_PROFILE_PATH 124 ); 125 Services.prefs.setBoolPref(SCHEDULED_BACKUPS_ENABLED_PREF_NAME, true); 126 Services.prefs.setIntPref(BACKUP_RETRY_LIMIT_PREF_NAME, RETRIES_FOR_TEST); 127 Services.prefs.setBoolPref(DISABLED_ON_IDLE_RETRY_PREF_NAME, false); 128 129 setupProfile(); 130 131 registerCleanupFunction(async () => { 132 Services.prefs.clearUserPref(BACKUP_DEFAULT_LOCATION_PREF_NAME); 133 Services.prefs.clearUserPref(SCHEDULED_BACKUPS_ENABLED_PREF_NAME); 134 Services.prefs.clearUserPref(BACKUP_RETRY_LIMIT_PREF_NAME); 135 Services.prefs.clearUserPref(DISABLED_ON_IDLE_RETRY_PREF_NAME); 136 137 await IOUtils.remove(TEST_PROFILE_PATH, { recursive: true }); 138 }); 139 }); 140 141 add_task(async function test_retries_no_backoff() { 142 Services.fog.testResetFOG(); 143 144 let bs = new BackupService(); 145 let sandbox = sinon.createSandbox(); 146 // Make createBackup fail intentionally 147 sandbox 148 .stub(bs, "resolveArchiveDestFolderPath") 149 .rejects(new BackupError("forced failure", ERRORS.UNKNOWN)); 150 151 // stub out idleDispatch 152 sandbox.stub(ChromeUtils, "idleDispatch").callsFake(callback => callback()); 153 154 sandbox.spy(bs, "createBackup"); 155 156 const n = Services.prefs.getIntPref(BACKUP_RETRY_LIMIT_PREF_NAME); 157 // now that we have an idle service, let's call create backup RETRY_LIMIT times 158 for (let i = 0; i <= n; i++) { 159 // ensure that there is no error code set 160 Services.prefs.setIntPref(BACKUP_ERROR_CODE_PREF_NAME, ERRORS.NONE); 161 162 // Set the lastBackupAttempt to the current backoff threshold, to avoid hitting 163 // the exponential backoff clause for this test. 164 Services.prefs.setStringPref( 165 BACKUP_DEBUG_INFO_PREF_NAME, 166 JSON.stringify({ 167 lastBackupAttempt: 168 Math.floor(Date.now() / 1000) - (BackupService.backoffSeconds() + 1), 169 errorCode: ERRORS.UNKNOWN, 170 lastRunStep: 0, 171 }) 172 ); 173 174 await create_backup_failure_expected_calls(bs, i + 1); 175 } 176 // check if it switched to no longer creating backups on idle 177 await create_backup_failure_expected_calls( 178 bs, 179 bs.createBackup.callCount, 180 "createBackup was not called since we hit the retry limit" 181 ); 182 183 sandbox.restore(); 184 }); 185 186 add_task(async function test_exponential_backoff() { 187 Services.fog.testResetFOG(); 188 189 let bs = new BackupService(); 190 let sandbox = sinon.createSandbox(); 191 const createBackupFailureStub = sandbox 192 .stub(bs, "resolveArchiveDestFolderPath") 193 .rejects(new BackupError("forced failure", ERRORS.UNKNOWN)); 194 195 sandbox.stub(ChromeUtils, "idleDispatch").callsFake(callback => callback()); 196 sandbox.spy(bs, "createBackup"); 197 198 Services.prefs.setIntPref(BACKUP_ERROR_CODE_PREF_NAME, ERRORS.NONE); 199 Services.prefs.setStringPref( 200 BACKUP_DEBUG_INFO_PREF_NAME, 201 JSON.stringify({ 202 lastBackupAttempt: 203 Math.floor(Date.now() / 1000) - (BackupService.backoffSeconds() + 1), 204 errorCode: ERRORS.UNKNOWN, 205 lastRunStep: 0, 206 }) 207 ); 208 209 Services.prefs.setIntPref(MINIMUM_TIME_BETWEEN_BACKUPS_SECONDS_PREF_NAME, 0); 210 registerCleanupFunction(() => { 211 Services.prefs.clearUserPref( 212 MINIMUM_TIME_BETWEEN_BACKUPS_SECONDS_PREF_NAME 213 ); 214 }); 215 216 await create_backup_failure_expected_calls(bs, 1); 217 218 // Remove the stub, ensure that a success leads to the prefs 219 // and retries resetting 220 createBackupFailureStub.restore(); 221 222 // eslint-disable-next-line mozilla/no-arbitrary-setTimeout 223 await new Promise(resolve => setTimeout(resolve, 10)); 224 225 let testProfilePath = await IOUtils.createUniqueDirectory( 226 PathUtils.tempDir, 227 "testBackup_profile" 228 ); 229 230 await bs.createBackup({ 231 profilePath: testProfilePath, 232 }); 233 234 Assert.equal( 235 Services.prefs.getIntPref(BACKUP_ERROR_CODE_PREF_NAME), 236 ERRORS.NONE, 237 "The error code is reset to NONE" 238 ); 239 240 Assert.equal( 241 60, 242 BackupService.backoffSeconds(), 243 "The exponential backoff is reset to 1 minute (60s)" 244 ); 245 246 Assert.ok( 247 !Services.prefs.getStringPref(BACKUP_DEBUG_INFO_PREF_NAME, null), 248 "Error debug info has been cleared" 249 ); 250 251 sandbox.restore(); 252 });