CacheIndex.h (51755B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #ifndef CacheIndex__h__ 6 #define CacheIndex__h__ 7 8 #include "CacheLog.h" 9 #include "CacheFileIOManager.h" 10 #include "nsIRunnable.h" 11 #include "CacheHashUtils.h" 12 #include "nsICacheStorageService.h" 13 #include "nsICacheEntry.h" 14 #include "nsILoadContextInfo.h" 15 #include "nsIWeakReferenceUtils.h" 16 #include "nsTHashtable.h" 17 #include "nsThreadUtils.h" 18 #include "mozilla/IntegerPrintfMacros.h" 19 #include "mozilla/SHA1.h" 20 #include "mozilla/StaticMutex.h" 21 #include "mozilla/StaticPtr.h" 22 #include "mozilla/EndianUtils.h" 23 #include "mozilla/TimeStamp.h" 24 #include "mozilla/UniquePtr.h" 25 #include "gtest/MozGtestFriend.h" 26 27 class nsIFile; 28 class nsIDirectoryEnumerator; 29 class nsITimer; 30 31 #ifdef DEBUG 32 # define DEBUG_STATS 1 33 #endif 34 35 namespace mozilla { 36 namespace net { 37 38 class CacheFileMetadata; 39 class FileOpenHelper; 40 class CacheIndexIterator; 41 42 const uint16_t kIndexTimeNotAvailable = 0xFFFFU; 43 const uint16_t kIndexTimeOutOfBound = 0xFFFEU; 44 45 using CacheIndexHeader = struct { 46 // Version of the index. The index must be ignored and deleted when the file 47 // on disk was written with a newer version. 48 uint32_t mVersion; 49 50 // Timestamp of time when the last successful write of the index started. 51 // During update process we use this timestamp for a quick validation of entry 52 // files. If last modified time of the file is lower than this timestamp, we 53 // skip parsing of such file since the information in index should be up to 54 // date. 55 uint32_t mTimeStamp; 56 57 // We set this flag as soon as possible after parsing index during startup 58 // and clean it after we write journal to disk during shutdown. We ignore the 59 // journal and start update process whenever this flag is set during index 60 // parsing. 61 uint32_t mIsDirty; 62 63 // The amount of data written to the cache. When it reaches 64 // kTelemetryReportBytesLimit a telemetry report is sent and the counter is 65 // reset. 66 uint32_t mKBWritten; 67 }; 68 69 static_assert(sizeof(CacheIndexHeader::mVersion) + 70 sizeof(CacheIndexHeader::mTimeStamp) + 71 sizeof(CacheIndexHeader::mIsDirty) + 72 sizeof(CacheIndexHeader::mKBWritten) == 73 sizeof(CacheIndexHeader), 74 "Unexpected sizeof(CacheIndexHeader)!"); 75 76 #pragma pack(push, 1) 77 struct CacheIndexRecord { 78 SHA1Sum::Hash mHash{}; 79 uint32_t mFrecency{0}; 80 OriginAttrsHash mOriginAttrsHash{0}; 81 uint16_t mOnStartTime{kIndexTimeNotAvailable}; 82 uint16_t mOnStopTime{kIndexTimeNotAvailable}; 83 uint8_t mContentType{nsICacheEntry::CONTENT_TYPE_UNKNOWN}; 84 85 /* 86 * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized 87 * 0100 0000 0000 0000 0000 0000 0000 0000 : anonymous 88 * 0010 0000 0000 0000 0000 0000 0000 0000 : removed 89 * 0001 0000 0000 0000 0000 0000 0000 0000 : dirty 90 * 0000 1000 0000 0000 0000 0000 0000 0000 : fresh 91 * 0000 0100 0000 0000 0000 0000 0000 0000 : pinned 92 * 0000 0010 0000 0000 0000 0000 0000 0000 : has cached alt data 93 * 0000 0001 0000 0000 0000 0000 0000 0000 : is a dictionary 94 * 0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB) 95 * Max file size is 16GiB 96 */ 97 uint32_t mFlags{0}; 98 99 CacheIndexRecord() = default; 100 }; 101 #pragma pack(pop) 102 103 // For Compression Dictionaries, we make special entries in the cache for 104 // each origin with a dictionary. In the data or metadata for each of 105 // these entries, we store the hashes of the dictionary, the match value 106 // (required), the match-dest value (optional), the id (optional) and the 107 // type (optional). 108 109 // We mark an entry if it's a dictionary (use-as-dictionary); if it is, 110 // when the entry is removed, we remove it from the origin's dictionary 111 // entry. If the origin's dictionary list is empty, we remove the origin. 112 113 static_assert(sizeof(CacheIndexRecord::mHash) + 114 sizeof(CacheIndexRecord::mFrecency) + 115 sizeof(CacheIndexRecord::mOriginAttrsHash) + 116 sizeof(CacheIndexRecord::mOnStartTime) + 117 sizeof(CacheIndexRecord::mOnStopTime) + 118 sizeof(CacheIndexRecord::mContentType) + 119 sizeof(CacheIndexRecord::mFlags) == 120 sizeof(CacheIndexRecord), 121 "Unexpected sizeof(CacheIndexRecord)!"); 122 123 class CacheIndexRecordWrapper final { 124 public: 125 NS_INLINE_DECL_THREADSAFE_REFCOUNTING_WITH_DESTROY( 126 CacheIndexRecordWrapper, DispatchDeleteSelfToCurrentThread()); 127 128 CacheIndexRecordWrapper() : mRec(MakeUnique<CacheIndexRecord>()) {} 129 CacheIndexRecord* Get() { return mRec.get(); } 130 131 private: 132 ~CacheIndexRecordWrapper(); 133 void DispatchDeleteSelfToCurrentThread(); 134 UniquePtr<CacheIndexRecord> mRec; 135 friend class DeleteCacheIndexRecordWrapper; 136 }; 137 138 class CacheIndexEntry : public PLDHashEntryHdr { 139 public: 140 using KeyType = const SHA1Sum::Hash&; 141 using KeyTypePointer = const SHA1Sum::Hash*; 142 143 explicit CacheIndexEntry(KeyTypePointer aKey) { 144 MOZ_COUNT_CTOR(CacheIndexEntry); 145 mRec = new CacheIndexRecordWrapper(); 146 LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]", 147 mRec->Get())); 148 memcpy(&mRec->Get()->mHash, aKey, sizeof(SHA1Sum::Hash)); 149 } 150 CacheIndexEntry(const CacheIndexEntry& aOther) { 151 MOZ_ASSERT_UNREACHABLE("CacheIndexEntry copy constructor is forbidden!"); 152 } 153 ~CacheIndexEntry() { 154 MOZ_COUNT_DTOR(CacheIndexEntry); 155 LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]", 156 mRec->Get())); 157 } 158 159 // KeyEquals(): does this entry match this key? 160 bool KeyEquals(KeyTypePointer aKey) const { 161 return memcmp(&mRec->Get()->mHash, aKey, sizeof(SHA1Sum::Hash)) == 0; 162 } 163 164 // KeyToPointer(): Convert KeyType to KeyTypePointer 165 static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; } 166 167 // HashKey(): calculate the hash number 168 static PLDHashNumber HashKey(KeyTypePointer aKey) { 169 return (reinterpret_cast<const uint32_t*>(aKey))[0]; 170 } 171 172 // ALLOW_MEMMOVE can we move this class with memmove(), or do we have 173 // to use the copy constructor? 174 enum { ALLOW_MEMMOVE = true }; 175 176 bool operator==(const CacheIndexEntry& aOther) const { 177 return KeyEquals(&aOther.mRec->Get()->mHash); 178 } 179 180 CacheIndexEntry& operator=(const CacheIndexEntry& aOther) { 181 MOZ_ASSERT(memcmp(&mRec->Get()->mHash, &aOther.mRec->Get()->mHash, 182 sizeof(SHA1Sum::Hash)) == 0); 183 mRec->Get()->mFrecency = aOther.mRec->Get()->mFrecency; 184 mRec->Get()->mOriginAttrsHash = aOther.mRec->Get()->mOriginAttrsHash; 185 mRec->Get()->mOnStartTime = aOther.mRec->Get()->mOnStartTime; 186 mRec->Get()->mOnStopTime = aOther.mRec->Get()->mOnStopTime; 187 mRec->Get()->mContentType = aOther.mRec->Get()->mContentType; 188 mRec->Get()->mFlags = aOther.mRec->Get()->mFlags; 189 return *this; 190 } 191 192 void InitNew() { 193 mRec->Get()->mFrecency = 0; 194 mRec->Get()->mOriginAttrsHash = 0; 195 mRec->Get()->mOnStartTime = kIndexTimeNotAvailable; 196 mRec->Get()->mOnStopTime = kIndexTimeNotAvailable; 197 mRec->Get()->mContentType = nsICacheEntry::CONTENT_TYPE_UNKNOWN; 198 mRec->Get()->mFlags = 0; 199 } 200 201 void Init(OriginAttrsHash aOriginAttrsHash, bool aAnonymous, bool aPinned) { 202 MOZ_ASSERT(mRec->Get()->mFrecency == 0); 203 MOZ_ASSERT(mRec->Get()->mOriginAttrsHash == 0); 204 MOZ_ASSERT(mRec->Get()->mOnStartTime == kIndexTimeNotAvailable); 205 MOZ_ASSERT(mRec->Get()->mOnStopTime == kIndexTimeNotAvailable); 206 MOZ_ASSERT(mRec->Get()->mContentType == 207 nsICacheEntry::CONTENT_TYPE_UNKNOWN); 208 // When we init the entry it must be fresh and may be dirty 209 MOZ_ASSERT((mRec->Get()->mFlags & ~kDirtyMask) == kFreshMask); 210 211 mRec->Get()->mOriginAttrsHash = aOriginAttrsHash; 212 mRec->Get()->mFlags |= kInitializedMask; 213 if (aAnonymous) { 214 mRec->Get()->mFlags |= kAnonymousMask; 215 } 216 if (aPinned) { 217 mRec->Get()->mFlags |= kPinnedMask; 218 } 219 } 220 221 const SHA1Sum::Hash* Hash() const { return &mRec->Get()->mHash; } 222 223 bool IsInitialized() const { 224 return !!(mRec->Get()->mFlags & kInitializedMask); 225 } 226 227 mozilla::net::OriginAttrsHash OriginAttrsHash() const { 228 return mRec->Get()->mOriginAttrsHash; 229 } 230 231 bool Anonymous() const { return !!(mRec->Get()->mFlags & kAnonymousMask); } 232 233 bool IsRemoved() const { return !!(mRec->Get()->mFlags & kRemovedMask); } 234 void MarkRemoved() { mRec->Get()->mFlags |= kRemovedMask; } 235 236 bool IsDirty() const { return !!(mRec->Get()->mFlags & kDirtyMask); } 237 void MarkDirty() { mRec->Get()->mFlags |= kDirtyMask; } 238 void ClearDirty() { mRec->Get()->mFlags &= ~kDirtyMask; } 239 240 bool IsFresh() const { return !!(mRec->Get()->mFlags & kFreshMask); } 241 void MarkFresh() { mRec->Get()->mFlags |= kFreshMask; } 242 243 bool IsPinned() const { return !!(mRec->Get()->mFlags & kPinnedMask); } 244 245 void SetFrecency(uint32_t aFrecency) { mRec->Get()->mFrecency = aFrecency; } 246 uint32_t GetFrecency() const { return mRec->Get()->mFrecency; } 247 248 void SetHasAltData(bool aHasAltData) { 249 aHasAltData ? mRec->Get()->mFlags |= kHasAltDataMask 250 : mRec->Get()->mFlags &= ~kHasAltDataMask; 251 } 252 bool GetHasAltData() const { 253 return !!(mRec->Get()->mFlags & kHasAltDataMask); 254 } 255 256 void SetOnStartTime(uint16_t aTime) { mRec->Get()->mOnStartTime = aTime; } 257 uint16_t GetOnStartTime() const { return mRec->Get()->mOnStartTime; } 258 259 void SetOnStopTime(uint16_t aTime) { mRec->Get()->mOnStopTime = aTime; } 260 uint16_t GetOnStopTime() const { return mRec->Get()->mOnStopTime; } 261 262 void SetContentType(uint8_t aType) { mRec->Get()->mContentType = aType; } 263 uint8_t GetContentType() const { return GetContentType(mRec->Get()); } 264 static uint8_t GetContentType(CacheIndexRecord* aRec) { 265 if (aRec->mContentType >= nsICacheEntry::CONTENT_TYPE_LAST) { 266 LOG( 267 ("CacheIndexEntry::GetContentType() - Found invalid content type " 268 "[hash=%08x%08x%08x%08x%08x, contentType=%u]", 269 LOGSHA1(aRec->mHash), aRec->mContentType)); 270 return nsICacheEntry::CONTENT_TYPE_UNKNOWN; 271 } 272 return aRec->mContentType; 273 } 274 275 // Sets filesize in kilobytes. 276 void SetFileSize(uint32_t aFileSize) { 277 if (aFileSize > kFileSizeMask) { 278 LOG( 279 ("CacheIndexEntry::SetFileSize() - FileSize is too large, " 280 "truncating to %u", 281 kFileSizeMask)); 282 aFileSize = kFileSizeMask; 283 } 284 mRec->Get()->mFlags &= ~kFileSizeMask; 285 mRec->Get()->mFlags |= aFileSize; 286 } 287 // Returns filesize in kilobytes. 288 uint32_t GetFileSize() const { return GetFileSize(*(mRec->Get())); } 289 static uint32_t GetFileSize(const CacheIndexRecord& aRec) { 290 return aRec.mFlags & kFileSizeMask; 291 } 292 static uint32_t IsPinned(CacheIndexRecord* aRec) { 293 return aRec->mFlags & kPinnedMask; 294 } 295 bool IsFileEmpty() const { return GetFileSize() == 0; } 296 297 void WriteToBuf(void* aBuf) { 298 uint8_t* ptr = static_cast<uint8_t*>(aBuf); 299 memcpy(ptr, mRec->Get()->mHash, sizeof(SHA1Sum::Hash)); 300 ptr += sizeof(SHA1Sum::Hash); 301 NetworkEndian::writeUint32(ptr, mRec->Get()->mFrecency); 302 ptr += sizeof(uint32_t); 303 NetworkEndian::writeUint64(ptr, mRec->Get()->mOriginAttrsHash); 304 ptr += sizeof(uint64_t); 305 NetworkEndian::writeUint16(ptr, mRec->Get()->mOnStartTime); 306 ptr += sizeof(uint16_t); 307 NetworkEndian::writeUint16(ptr, mRec->Get()->mOnStopTime); 308 ptr += sizeof(uint16_t); 309 *ptr = mRec->Get()->mContentType; 310 ptr += sizeof(uint8_t); 311 // Dirty and fresh flags should never go to disk, since they make sense only 312 // during current session. 313 NetworkEndian::writeUint32( 314 ptr, mRec->Get()->mFlags & ~(kDirtyMask | kFreshMask)); 315 } 316 317 void ReadFromBuf(void* aBuf) { 318 const uint8_t* ptr = static_cast<const uint8_t*>(aBuf); 319 MOZ_ASSERT(memcmp(&mRec->Get()->mHash, ptr, sizeof(SHA1Sum::Hash)) == 0); 320 ptr += sizeof(SHA1Sum::Hash); 321 mRec->Get()->mFrecency = NetworkEndian::readUint32(ptr); 322 ptr += sizeof(uint32_t); 323 mRec->Get()->mOriginAttrsHash = NetworkEndian::readUint64(ptr); 324 ptr += sizeof(uint64_t); 325 mRec->Get()->mOnStartTime = NetworkEndian::readUint16(ptr); 326 ptr += sizeof(uint16_t); 327 mRec->Get()->mOnStopTime = NetworkEndian::readUint16(ptr); 328 ptr += sizeof(uint16_t); 329 mRec->Get()->mContentType = *ptr; 330 ptr += sizeof(uint8_t); 331 mRec->Get()->mFlags = NetworkEndian::readUint32(ptr); 332 } 333 334 void Log() const { 335 LOG( 336 ("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u," 337 " initialized=%u, removed=%u, dirty=%u, anonymous=%u, " 338 "originAttrsHash=%" PRIx64 ", frecency=%u, hasAltData=%u, " 339 "onStartTime=%u, onStopTime=%u, contentType=%u, size=%u]", 340 this, LOGSHA1(mRec->Get()->mHash), IsFresh(), IsInitialized(), 341 IsRemoved(), IsDirty(), Anonymous(), OriginAttrsHash(), GetFrecency(), 342 GetHasAltData(), GetOnStartTime(), GetOnStopTime(), GetContentType(), 343 GetFileSize())); 344 } 345 346 static bool RecordMatchesLoadContextInfo(CacheIndexRecordWrapper* aRec, 347 nsILoadContextInfo* aInfo) { 348 MOZ_ASSERT(aInfo); 349 350 return !aInfo->IsPrivate() && 351 GetOriginAttrsHash(*aInfo->OriginAttributesPtr()) == 352 aRec->Get()->mOriginAttrsHash && 353 aInfo->IsAnonymous() == !!(aRec->Get()->mFlags & kAnonymousMask); 354 } 355 356 // Memory reporting 357 size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const { 358 return mallocSizeOf(mRec->Get()); 359 } 360 361 size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const { 362 return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf); 363 } 364 365 private: 366 friend class CacheIndexEntryUpdate; 367 friend class CacheIndex; 368 friend class CacheIndexEntryAutoManage; 369 friend struct CacheIndexRecord; 370 371 static const uint32_t kInitializedMask = 0x80000000; 372 static const uint32_t kAnonymousMask = 0x40000000; 373 374 // This flag is set when the entry was removed. We need to keep this 375 // information in memory until we write the index file. 376 static const uint32_t kRemovedMask = 0x20000000; 377 378 // This flag is set when the information in memory is not in sync with the 379 // information in index file on disk. 380 static const uint32_t kDirtyMask = 0x10000000; 381 382 // This flag is set when the information about the entry is fresh, i.e. 383 // we've created or opened this entry during this session, or we've seen 384 // this entry during update or build process. 385 static const uint32_t kFreshMask = 0x08000000; 386 387 // Indicates a pinned entry. 388 static const uint32_t kPinnedMask = 0x04000000; 389 390 // Indicates there is cached alternative data in the entry. 391 static const uint32_t kHasAltDataMask = 0x02000000; 392 393 // Indicates that this entry is a dictionary 394 static const uint32_t kDictionaryMask = 0x01000000; 395 396 // FileSize in kilobytes (max 16GB) 397 static const uint32_t kFileSizeMask = 0x00FFFFFF; 398 399 RefPtr<CacheIndexRecordWrapper> mRec; 400 }; 401 402 class CacheIndexEntryUpdate : public CacheIndexEntry { 403 public: 404 explicit CacheIndexEntryUpdate(CacheIndexEntry::KeyTypePointer aKey) 405 : CacheIndexEntry(aKey), mUpdateFlags(0) { 406 MOZ_COUNT_CTOR(CacheIndexEntryUpdate); 407 LOG(("CacheIndexEntryUpdate::CacheIndexEntryUpdate()")); 408 } 409 ~CacheIndexEntryUpdate() { 410 MOZ_COUNT_DTOR(CacheIndexEntryUpdate); 411 LOG(("CacheIndexEntryUpdate::~CacheIndexEntryUpdate()")); 412 } 413 414 CacheIndexEntryUpdate& operator=(const CacheIndexEntry& aOther) { 415 MOZ_ASSERT(memcmp(&mRec->Get()->mHash, &aOther.mRec->Get()->mHash, 416 sizeof(SHA1Sum::Hash)) == 0); 417 mUpdateFlags = 0; 418 *(static_cast<CacheIndexEntry*>(this)) = aOther; 419 return *this; 420 } 421 422 void InitNew() { 423 mUpdateFlags = kFrecencyUpdatedMask | kHasAltDataUpdatedMask | 424 kOnStartTimeUpdatedMask | kOnStopTimeUpdatedMask | 425 kContentTypeUpdatedMask | kFileSizeUpdatedMask; 426 CacheIndexEntry::InitNew(); 427 } 428 429 void SetFrecency(uint32_t aFrecency) { 430 mUpdateFlags |= kFrecencyUpdatedMask; 431 CacheIndexEntry::SetFrecency(aFrecency); 432 } 433 434 void SetHasAltData(bool aHasAltData) { 435 mUpdateFlags |= kHasAltDataUpdatedMask; 436 CacheIndexEntry::SetHasAltData(aHasAltData); 437 } 438 439 void SetOnStartTime(uint16_t aTime) { 440 mUpdateFlags |= kOnStartTimeUpdatedMask; 441 CacheIndexEntry::SetOnStartTime(aTime); 442 } 443 444 void SetOnStopTime(uint16_t aTime) { 445 mUpdateFlags |= kOnStopTimeUpdatedMask; 446 CacheIndexEntry::SetOnStopTime(aTime); 447 } 448 449 void SetContentType(uint8_t aType) { 450 mUpdateFlags |= kContentTypeUpdatedMask; 451 CacheIndexEntry::SetContentType(aType); 452 } 453 454 void SetFileSize(uint32_t aFileSize) { 455 mUpdateFlags |= kFileSizeUpdatedMask; 456 CacheIndexEntry::SetFileSize(aFileSize); 457 } 458 459 void ApplyUpdate(CacheIndexEntry* aDst) { 460 MOZ_ASSERT(memcmp(&mRec->Get()->mHash, &aDst->mRec->Get()->mHash, 461 sizeof(SHA1Sum::Hash)) == 0); 462 if (mUpdateFlags & kFrecencyUpdatedMask) { 463 aDst->mRec->Get()->mFrecency = mRec->Get()->mFrecency; 464 } 465 aDst->mRec->Get()->mOriginAttrsHash = mRec->Get()->mOriginAttrsHash; 466 if (mUpdateFlags & kOnStartTimeUpdatedMask) { 467 aDst->mRec->Get()->mOnStartTime = mRec->Get()->mOnStartTime; 468 } 469 if (mUpdateFlags & kOnStopTimeUpdatedMask) { 470 aDst->mRec->Get()->mOnStopTime = mRec->Get()->mOnStopTime; 471 } 472 if (mUpdateFlags & kContentTypeUpdatedMask) { 473 aDst->mRec->Get()->mContentType = mRec->Get()->mContentType; 474 } 475 if (mUpdateFlags & kHasAltDataUpdatedMask && 476 ((aDst->mRec->Get()->mFlags ^ mRec->Get()->mFlags) & kHasAltDataMask)) { 477 // Toggle the bit if we need to. 478 aDst->mRec->Get()->mFlags ^= kHasAltDataMask; 479 } 480 481 if (mUpdateFlags & kFileSizeUpdatedMask) { 482 // Copy all flags except |HasAltData|. 483 aDst->mRec->Get()->mFlags |= (mRec->Get()->mFlags & ~kHasAltDataMask); 484 } else { 485 // Copy all flags except |HasAltData| and file size. 486 aDst->mRec->Get()->mFlags &= kFileSizeMask; 487 aDst->mRec->Get()->mFlags |= 488 (mRec->Get()->mFlags & ~kHasAltDataMask & ~kFileSizeMask); 489 } 490 } 491 492 private: 493 static const uint32_t kFrecencyUpdatedMask = 0x00000001; 494 static const uint32_t kContentTypeUpdatedMask = 0x00000002; 495 static const uint32_t kFileSizeUpdatedMask = 0x00000004; 496 static const uint32_t kHasAltDataUpdatedMask = 0x00000008; 497 static const uint32_t kOnStartTimeUpdatedMask = 0x00000010; 498 static const uint32_t kOnStopTimeUpdatedMask = 0x00000020; 499 500 uint32_t mUpdateFlags; 501 }; 502 503 class CacheIndexStats { 504 public: 505 CacheIndexStats() { 506 for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) { 507 mCountByType[i] = 0; 508 mSizeByType[i] = 0; 509 } 510 } 511 512 bool operator==(const CacheIndexStats& aOther) const { 513 for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) { 514 if (mCountByType[i] != aOther.mCountByType[i] || 515 mSizeByType[i] != aOther.mSizeByType[i]) { 516 return false; 517 } 518 } 519 520 return 521 #ifdef DEBUG 522 aOther.mStateLogged == mStateLogged && 523 #endif 524 aOther.mCount == mCount && aOther.mNotInitialized == mNotInitialized && 525 aOther.mRemoved == mRemoved && aOther.mDirty == mDirty && 526 aOther.mFresh == mFresh && aOther.mEmpty == mEmpty && 527 aOther.mSize == mSize; 528 } 529 530 #ifdef DEBUG 531 void DisableLogging() { mDisableLogging = true; } 532 #endif 533 534 void Log() { 535 LOG( 536 ("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, " 537 "dirty=%u, fresh=%u, empty=%u, size=%u]", 538 mCount, mNotInitialized, mRemoved, mDirty, mFresh, mEmpty, mSize)); 539 } 540 541 void Clear() { 542 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Clear() - state logged!"); 543 544 mCount = 0; 545 mNotInitialized = 0; 546 mRemoved = 0; 547 mDirty = 0; 548 mFresh = 0; 549 mEmpty = 0; 550 mSize = 0; 551 for (uint32_t i = 0; i < nsICacheEntry::CONTENT_TYPE_LAST; ++i) { 552 mCountByType[i] = 0; 553 mSizeByType[i] = 0; 554 } 555 } 556 557 #ifdef DEBUG 558 bool StateLogged() { return mStateLogged; } 559 #endif 560 561 uint32_t Count() { 562 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Count() - state logged!"); 563 return mCount; 564 } 565 566 uint32_t CountByType(uint8_t aContentType) { 567 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::CountByType() - state logged!"); 568 MOZ_RELEASE_ASSERT(aContentType < nsICacheEntry::CONTENT_TYPE_LAST); 569 return mCountByType[aContentType]; 570 } 571 572 uint32_t Dirty() { 573 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Dirty() - state logged!"); 574 return mDirty; 575 } 576 577 uint32_t Fresh() { 578 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Fresh() - state logged!"); 579 return mFresh; 580 } 581 582 uint32_t ActiveEntriesCount() { 583 MOZ_ASSERT(!mStateLogged, 584 "CacheIndexStats::ActiveEntriesCount() - state " 585 "logged!"); 586 return mCount - mRemoved - mNotInitialized - mEmpty; 587 } 588 589 uint32_t Size() { 590 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!"); 591 return mSize; 592 } 593 594 uint32_t SizeByType(uint8_t aContentType) { 595 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::SizeByType() - state logged!"); 596 MOZ_RELEASE_ASSERT(aContentType < nsICacheEntry::CONTENT_TYPE_LAST); 597 return mSizeByType[aContentType]; 598 } 599 600 void BeforeChange(const CacheIndexEntry* aEntry) { 601 #ifdef DEBUG_STATS 602 if (!mDisableLogging) { 603 LOG(("CacheIndexStats::BeforeChange()")); 604 Log(); 605 } 606 #endif 607 608 MOZ_ASSERT(!mStateLogged, 609 "CacheIndexStats::BeforeChange() - state " 610 "logged!"); 611 #ifdef DEBUG 612 mStateLogged = true; 613 #endif 614 if (aEntry) { 615 MOZ_ASSERT(mCount); 616 uint8_t contentType = aEntry->GetContentType(); 617 mCount--; 618 mCountByType[contentType]--; 619 if (aEntry->IsDirty()) { 620 MOZ_ASSERT(mDirty); 621 mDirty--; 622 } 623 if (aEntry->IsFresh()) { 624 MOZ_ASSERT(mFresh); 625 mFresh--; 626 } 627 if (aEntry->IsRemoved()) { 628 MOZ_ASSERT(mRemoved); 629 mRemoved--; 630 } else { 631 if (!aEntry->IsInitialized()) { 632 MOZ_ASSERT(mNotInitialized); 633 mNotInitialized--; 634 } else { 635 if (aEntry->IsFileEmpty()) { 636 MOZ_ASSERT(mEmpty); 637 mEmpty--; 638 } else { 639 MOZ_ASSERT(mSize >= aEntry->GetFileSize()); 640 mSize -= aEntry->GetFileSize(); 641 mSizeByType[contentType] -= aEntry->GetFileSize(); 642 } 643 } 644 } 645 } 646 } 647 648 void AfterChange(const CacheIndexEntry* aEntry) { 649 MOZ_ASSERT(mStateLogged, 650 "CacheIndexStats::AfterChange() - state not " 651 "logged!"); 652 #ifdef DEBUG 653 mStateLogged = false; 654 #endif 655 if (aEntry) { 656 uint8_t contentType = aEntry->GetContentType(); 657 ++mCount; 658 ++mCountByType[contentType]; 659 if (aEntry->IsDirty()) { 660 mDirty++; 661 } 662 if (aEntry->IsFresh()) { 663 mFresh++; 664 } 665 if (aEntry->IsRemoved()) { 666 mRemoved++; 667 } else { 668 if (!aEntry->IsInitialized()) { 669 mNotInitialized++; 670 } else { 671 if (aEntry->IsFileEmpty()) { 672 mEmpty++; 673 } else { 674 mSize += aEntry->GetFileSize(); 675 mSizeByType[contentType] += aEntry->GetFileSize(); 676 } 677 } 678 } 679 } 680 681 #ifdef DEBUG_STATS 682 if (!mDisableLogging) { 683 LOG(("CacheIndexStats::AfterChange()")); 684 Log(); 685 } 686 #endif 687 } 688 689 private: 690 uint32_t mCount{0}; 691 uint32_t mCountByType[nsICacheEntry::CONTENT_TYPE_LAST]{0}; 692 uint32_t mNotInitialized{0}; 693 uint32_t mRemoved{0}; 694 uint32_t mDirty{0}; 695 uint32_t mFresh{0}; 696 uint32_t mEmpty{0}; 697 uint32_t mSize{0}; 698 uint32_t mSizeByType[nsICacheEntry::CONTENT_TYPE_LAST]{0}; 699 #ifdef DEBUG 700 // We completely remove the data about an entry from the stats in 701 // BeforeChange() and set this flag to true. The entry is then modified, 702 // deleted or created and the data is again put into the stats and this flag 703 // set to false. Statistics must not be read during this time since the 704 // information is not correct. 705 bool mStateLogged{false}; 706 707 // Disables logging in this instance of CacheIndexStats 708 bool mDisableLogging{false}; 709 #endif 710 }; 711 712 class CacheIndex final : public CacheFileIOListener, public nsIRunnable { 713 public: 714 NS_DECL_THREADSAFE_ISUPPORTS 715 NS_DECL_NSIRUNNABLE 716 717 CacheIndex(); 718 FRIEND_TEST(FrecencyStorageTest, AppendRemoveRecordTest); 719 FRIEND_TEST(FrecencyStorageTest, ReplaceRecordTest); 720 FRIEND_TEST(FrecencyStorageTest, ClearTest); 721 FRIEND_TEST(FrecencyStorageTest, GetSortedSnapshotForEvictionTest); 722 FRIEND_TEST(FrecencyStorageTest, PerformanceTest); 723 724 static nsresult Init(nsIFile* aCacheDirectory); 725 static nsresult PreShutdown(); 726 static nsresult Shutdown(); 727 728 // Following methods can be called only on IO thread. 729 730 // Add entry to the index. The entry shouldn't be present in index. This 731 // method is called whenever a new handle for a new entry file is created. The 732 // newly created entry is not initialized and it must be either initialized 733 // with InitEntry() or removed with RemoveEntry(). 734 static nsresult AddEntry(const SHA1Sum::Hash* aHash); 735 736 // Inform index about an existing entry that should be present in index. This 737 // method is called whenever a new handle for an existing entry file is 738 // created. Like in case of AddEntry(), either InitEntry() or RemoveEntry() 739 // must be called on the entry, since the entry is not initizlized if the 740 // index is outdated. 741 static nsresult EnsureEntryExists(const SHA1Sum::Hash* aHash); 742 743 // Initialize the entry. It MUST be present in index. Call to AddEntry() or 744 // EnsureEntryExists() must precede the call to this method. 745 static nsresult InitEntry(const SHA1Sum::Hash* aHash, 746 OriginAttrsHash aOriginAttrsHash, bool aAnonymous, 747 bool aPinned); 748 749 // Remove entry from index. The entry should be present in index. 750 static nsresult RemoveEntry(const SHA1Sum::Hash* aHash, 751 const nsACString& aKey, 752 bool aClearDictionary = true); 753 754 // Update some information in entry. The entry MUST be present in index and 755 // MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to 756 // InitEntry() must precede the call to this method. 757 // Pass nullptr if the value didn't change. 758 static nsresult UpdateEntry(const SHA1Sum::Hash* aHash, 759 const uint32_t* aFrecency, 760 const bool* aHasAltData, 761 const uint16_t* aOnStartTime, 762 const uint16_t* aOnStopTime, 763 const uint8_t* aContentType, 764 const uint32_t* aSize); 765 766 // Mark entries so we won't find them. Used to implement synchronous 767 // clearing for Clear-Site-Data: cache for Compression Dictionaries 768 static void EvictByContext(const nsAString& aOrigin, 769 const nsAString& aBaseDomain); 770 771 // Remove all entries from the index. Called when clearing the whole cache. 772 static nsresult RemoveAll(); 773 774 enum EntryStatus { EXISTS = 0, DOES_NOT_EXIST = 1, DO_NOT_KNOW = 2 }; 775 776 // Used to store a snapshot of the frecency storage 777 using EvictionSortedSnapshot = nsTArray<RefPtr<CacheIndexRecordWrapper>>; 778 779 // Returns status of the entry in index for the given key. It can be called 780 // on any thread. 781 // If the optional aCB callback is given, the it will be called with a 782 // CacheIndexEntry only if _retval is EXISTS when the method returns. 783 static nsresult HasEntry( 784 const nsACString& aKey, EntryStatus* _retval, 785 const std::function<void(const CacheIndexEntry*)>& aCB = nullptr); 786 static nsresult HasEntry( 787 const SHA1Sum::Hash& hash, EntryStatus* _retval, 788 const std::function<void(const CacheIndexEntry*)>& aCB = nullptr); 789 790 // Returns a hash of the least important entry that should be evicted if the 791 // cache size is over limit and also returns a total number of all entries in 792 // the index minus the number of forced valid entries and unpinned entries 793 // that we encounter when searching (see below) 794 static nsresult GetEntryForEviction(EvictionSortedSnapshot& aSnapshot, 795 bool aIgnoreEmptyEntries, 796 SHA1Sum::Hash* aHash, uint32_t* aCnt); 797 798 // Returns a sorted snapshot of the frecency storage. 799 static EvictionSortedSnapshot GetSortedSnapshotForEviction(); 800 801 // Checks if a cache entry is currently forced valid. Used to prevent an entry 802 // (that has been forced valid) from being evicted when the cache size reaches 803 // its limit. 804 static bool IsForcedValidEntry(const SHA1Sum::Hash* aHash); 805 806 // Returns cache size in kB. 807 static nsresult GetCacheSize(uint32_t* _retval); 808 809 // Returns number of entry files in the cache 810 static nsresult GetEntryFileCount(uint32_t* _retval); 811 812 // Synchronously returns the disk occupation and number of entries 813 // per-context. Callable on any thread. It will ignore loadContextInfo and get 814 // stats for all entries if the aInfo is a nullptr. 815 static nsresult GetCacheStats(nsILoadContextInfo* aInfo, uint32_t* aSize, 816 uint32_t* aCount); 817 818 // Asynchronously gets the disk cache size, used for display in the UI. 819 static nsresult AsyncGetDiskConsumption( 820 nsICacheStorageConsumptionObserver* aObserver); 821 822 // Returns an iterator that returns entries matching a given context that were 823 // present in the index at the time this method was called. If aAddNew is true 824 // then the iterator will also return entries created after this call. 825 // NOTE: When some entry is removed from index it is removed also from the 826 // iterator regardless what aAddNew was passed. 827 static nsresult GetIterator(nsILoadContextInfo* aInfo, bool aAddNew, 828 CacheIndexIterator** _retval); 829 830 // Returns true if we _think_ that the index is up to date. I.e. the state is 831 // READY or WRITING and mIndexNeedsUpdate as well as mShuttingDown is false. 832 static nsresult IsUpToDate(bool* _retval); 833 834 // Called from CacheStorageService::Clear() and 835 // CacheFileContextEvictor::EvictEntries(), sets a flag that blocks 836 // notification to AsyncGetDiskConsumption. 837 static void OnAsyncEviction(bool aEvicting); 838 839 // We keep track of total bytes written to the cache to be able to do 840 // a telemetry report after writting certain amount of data to the cache. 841 static void UpdateTotalBytesWritten(uint32_t aBytesWritten); 842 843 // Memory reporting 844 static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf); 845 static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf); 846 847 private: 848 friend class CacheIndexEntryAutoManage; 849 friend class FileOpenHelper; 850 friend class CacheIndexIterator; 851 friend class CacheIndexRecordWrapper; 852 friend class DeleteCacheIndexRecordWrapper; 853 854 virtual ~CacheIndex(); 855 856 NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) override; 857 void OnFileOpenedInternal(FileOpenHelper* aOpener, CacheFileHandle* aHandle, 858 nsresult aResult, 859 const StaticMutexAutoLock& aProofOfLock) 860 MOZ_REQUIRES(sLock); 861 NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf, 862 nsresult aResult) override; 863 NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf, 864 nsresult aResult) override; 865 NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) override; 866 NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) override; 867 NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle, nsresult aResult) override; 868 869 nsresult InitInternal(nsIFile* aCacheDirectory, 870 const StaticMutexAutoLock& aProofOfLock); 871 void PreShutdownInternal(); 872 873 // This method returns false when index is not initialized or is shut down. 874 bool IsIndexUsable() MOZ_REQUIRES(sLock); 875 876 // This method checks whether the entry has the same values of 877 // originAttributes and isAnonymous. We don't expect to find a collision 878 // since these values are part of the key that we hash and we use a strong 879 // hash function. 880 static bool IsCollision(CacheIndexEntry* aEntry, 881 OriginAttrsHash aOriginAttrsHash, bool aAnonymous); 882 883 // Checks whether any of the information about the entry has changed. 884 static bool HasEntryChanged(CacheIndexEntry* aEntry, 885 const uint32_t* aFrecency, 886 const bool* aHasAltData, 887 const uint16_t* aOnStartTime, 888 const uint16_t* aOnStopTime, 889 const uint8_t* aContentType, 890 const uint32_t* aSize); 891 892 // Merge all pending operations from mPendingUpdates into mIndex. 893 void ProcessPendingOperations(const StaticMutexAutoLock& aProofOfLock) 894 MOZ_REQUIRES(sLock); 895 896 // Following methods perform writing of the index file. 897 // 898 // The index is written periodically, but not earlier than once in 899 // kMinDumpInterval and there must be at least kMinUnwrittenChanges 900 // differences between index on disk and in memory. Index is always first 901 // written to a temporary file and the old index file is replaced when the 902 // writing process succeeds. 903 // 904 // Starts writing of index when both limits (minimal delay between writes and 905 // minimum number of changes in index) were exceeded. 906 bool WriteIndexToDiskIfNeeded(const StaticMutexAutoLock& aProofOfLock) 907 MOZ_REQUIRES(sLock); 908 // Starts writing of index file. 909 void WriteIndexToDisk(const StaticMutexAutoLock& aProofOfLock) 910 MOZ_REQUIRES(sLock); 911 // Serializes part of mIndex hashtable to the write buffer a writes the buffer 912 // to the file. 913 void WriteRecords(const StaticMutexAutoLock& aProofOfLock) 914 MOZ_REQUIRES(sLock); 915 // Finalizes writing process. 916 void FinishWrite(bool aSucceeded, const StaticMutexAutoLock& aProofOfLock) 917 MOZ_REQUIRES(sLock); 918 919 // Following methods perform writing of the journal during shutdown. All these 920 // methods must be called only during shutdown since they write/delete files 921 // directly on the main thread instead of using CacheFileIOManager that does 922 // it asynchronously on IO thread. Journal contains only entries that are 923 // dirty, i.e. changes that are not present in the index file on the disk. 924 // When the log is written successfully, the dirty flag in index file is 925 // cleared. 926 nsresult GetFile(const nsACString& aName, nsIFile** _retval); 927 void RemoveFile(const nsACString& aName) MOZ_REQUIRES(sLock); 928 void RemoveAllIndexFiles() MOZ_REQUIRES(sLock); 929 void RemoveJournalAndTempFile() MOZ_REQUIRES(sLock); 930 // Writes journal to the disk and clears dirty flag in index header. 931 nsresult WriteLogToDisk() MOZ_REQUIRES(sLock); 932 933 // Following methods perform reading of the index from the disk. 934 // 935 // Index is read at startup just after initializing the CacheIndex. There are 936 // 3 files used when manipulating with index: index file, journal file and 937 // a temporary file. All files contain the hash of the data, so we can check 938 // whether the content is valid and complete. Index file contains also a dirty 939 // flag in the index header which is unset on a clean shutdown. During opening 940 // and reading of the files we determine the status of the whole index from 941 // the states of the separate files. Following table shows all possible 942 // combinations: 943 // 944 // index, journal, tmpfile 945 // M * * - index is missing -> BUILD 946 // I * * - index is invalid -> BUILD 947 // D * * - index is dirty -> UPDATE 948 // C M * - index is dirty -> UPDATE 949 // C I * - unexpected state -> UPDATE 950 // C V E - unexpected state -> UPDATE 951 // C V M - index is up to date -> READY 952 // 953 // where the letters mean: 954 // * - any state 955 // E - file exists 956 // M - file is missing 957 // I - data is invalid (parsing failed or hash didn't match) 958 // D - dirty (data in index file is correct, but dirty flag is set) 959 // C - clean (index file is clean) 960 // V - valid (data in journal file is correct) 961 // 962 // Note: We accept the data from journal only when the index is up to date as 963 // a whole (i.e. C,V,M state). 964 // 965 // We rename the journal file to the temporary file as soon as possible after 966 // initial test to ensure that we start update process on the next startup if 967 // FF crashes during parsing of the index. 968 // 969 // Initiates reading index from disk. 970 void ReadIndexFromDisk(const StaticMutexAutoLock& aProofOfLock) 971 MOZ_REQUIRES(sLock); 972 // Starts reading data from index file. 973 void StartReadingIndex(const StaticMutexAutoLock& aProofOfLock) 974 MOZ_REQUIRES(sLock); 975 // Parses data read from index file. 976 void ParseRecords(const StaticMutexAutoLock& aProofOfLock) 977 MOZ_REQUIRES(sLock); 978 // Starts reading data from journal file. 979 void StartReadingJournal(const StaticMutexAutoLock& aProofOfLock) 980 MOZ_REQUIRES(sLock); 981 // Parses data read from journal file. 982 void ParseJournal(const StaticMutexAutoLock& aProofOfLock) 983 MOZ_REQUIRES(sLock); 984 // Merges entries from journal into mIndex. 985 void MergeJournal(const StaticMutexAutoLock& aProofOfLock) 986 MOZ_REQUIRES(sLock); 987 // In debug build this method checks that we have no fresh entry in mIndex 988 // after we finish reading index and before we process pending operations. 989 void EnsureNoFreshEntry() MOZ_REQUIRES(sLock); 990 // In debug build this method is called after processing pending operations 991 // to make sure mIndexStats contains correct information. 992 void EnsureCorrectStats() MOZ_REQUIRES(sLock); 993 994 // Finalizes reading process. 995 void FinishRead(bool aSucceeded, const StaticMutexAutoLock& aProofOfLock) 996 MOZ_REQUIRES(sLock); 997 998 // Following methods perform updating and building of the index. 999 // Timer callback that starts update or build process. 1000 static void DelayedUpdate(nsITimer* aTimer, void* aClosure); 1001 void DelayedUpdateLocked(const StaticMutexAutoLock& aProofOfLock) 1002 MOZ_REQUIRES(sLock); 1003 // Posts timer event that start update or build process. 1004 nsresult ScheduleUpdateTimer(uint32_t aDelay) MOZ_REQUIRES(sLock); 1005 nsresult SetupDirectoryEnumerator() MOZ_REQUIRES(sLock); 1006 nsresult InitEntryFromDiskData(CacheIndexEntry* aEntry, 1007 CacheFileMetadata* aMetaData, 1008 int64_t aFileSize); 1009 // Returns true when either a timer is scheduled or event is posted. 1010 bool IsUpdatePending() MOZ_REQUIRES(sLock); 1011 // Iterates through all files in entries directory that we didn't create/open 1012 // during this session, parses them and adds the entries to the index. 1013 void BuildIndex(const StaticMutexAutoLock& aProofOfLock) MOZ_REQUIRES(sLock); 1014 1015 bool StartUpdatingIndexIfNeeded(const StaticMutexAutoLock& aProofOfLock, 1016 bool aSwitchingToReadyState = false); 1017 // Starts update or build process or fires a timer when it is too early after 1018 // startup. 1019 void StartUpdatingIndex(bool aRebuild, 1020 const StaticMutexAutoLock& aProofOfLock) 1021 MOZ_REQUIRES(sLock); 1022 // Iterates through all files in entries directory that we didn't create/open 1023 // during this session and theirs last modified time is newer than timestamp 1024 // in the index header. Parses the files and adds the entries to the index. 1025 void UpdateIndex(const StaticMutexAutoLock& aProofOfLock) MOZ_REQUIRES(sLock); 1026 // Finalizes update or build process. 1027 void FinishUpdate(bool aSucceeded, const StaticMutexAutoLock& aProofOfLock) 1028 MOZ_REQUIRES(sLock); 1029 1030 void RemoveNonFreshEntries(const StaticMutexAutoLock& aProofOfLock) 1031 MOZ_REQUIRES(sLock); 1032 1033 enum EState { 1034 // Initial state in which the index is not usable 1035 // Possible transitions: 1036 // -> READING 1037 INITIAL = 0, 1038 1039 // Index is being read from the disk. 1040 // Possible transitions: 1041 // -> INITIAL - We failed to dispatch a read event. 1042 // -> BUILDING - No or corrupted index file was found. 1043 // -> UPDATING - No or corrupted journal file was found. 1044 // - Dirty flag was set in index header. 1045 // -> READY - Index was read successfully or was interrupted by 1046 // pre-shutdown. 1047 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. 1048 READING = 1, 1049 1050 // Index is being written to the disk. 1051 // Possible transitions: 1052 // -> READY - Writing of index finished or was interrupted by 1053 // pre-shutdown.. 1054 // -> UPDATING - Writing of index finished, but index was found outdated 1055 // during writing. 1056 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. 1057 WRITING = 2, 1058 1059 // Index is being build. 1060 // Possible transitions: 1061 // -> READY - Building of index finished or was interrupted by 1062 // pre-shutdown. 1063 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. 1064 BUILDING = 3, 1065 1066 // Index is being updated. 1067 // Possible transitions: 1068 // -> READY - Updating of index finished or was interrupted by 1069 // pre-shutdown. 1070 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. 1071 UPDATING = 4, 1072 1073 // Index is ready. 1074 // Possible transitions: 1075 // -> UPDATING - Index was found outdated. 1076 // -> SHUTDOWN - Index is shutting down. 1077 READY = 5, 1078 1079 // Index is shutting down. 1080 SHUTDOWN = 6 1081 }; 1082 1083 static char const* StateString(EState aState); 1084 void ChangeState(EState aNewState, const StaticMutexAutoLock& aProofOfLock); 1085 void NotifyAsyncGetDiskConsumptionCallbacks() MOZ_REQUIRES(sLock); 1086 1087 // Allocates and releases buffer used for reading and writing index. 1088 void AllocBuffer() MOZ_REQUIRES(sLock); 1089 void ReleaseBuffer() MOZ_REQUIRES(sLock); 1090 1091 // Methods used by CacheIndexEntryAutoManage to keep the iterators up to date. 1092 void AddRecordToIterators(CacheIndexRecordWrapper* aRecord, 1093 const StaticMutexAutoLock& aProofOfLock) 1094 MOZ_REQUIRES(sLock); 1095 void RemoveRecordFromIterators(CacheIndexRecordWrapper* aRecord, 1096 const StaticMutexAutoLock& aProofOfLock) 1097 MOZ_REQUIRES(sLock); 1098 void ReplaceRecordInIterators(CacheIndexRecordWrapper* aOldRecord, 1099 CacheIndexRecordWrapper* aNewRecord, 1100 const StaticMutexAutoLock& aProofOfLock) 1101 MOZ_REQUIRES(sLock); 1102 1103 // Memory reporting (private part) 1104 size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const 1105 MOZ_REQUIRES(sLock); 1106 1107 // Reports telemetry about cache, i.e. size, entry count and content type 1108 // stats. 1109 void DoTelemetryReport() MOZ_REQUIRES(sLock); 1110 1111 static mozilla::StaticRefPtr<CacheIndex> gInstance MOZ_GUARDED_BY(sLock); 1112 1113 // sLock guards almost everything here... 1114 // Also guards FileOpenHelper::mCanceled 1115 static StaticMutex sLock; 1116 1117 nsCOMPtr<nsIFile> mCacheDirectory; 1118 1119 EState mState MOZ_GUARDED_BY(sLock){INITIAL}; 1120 // Timestamp of time when the index was initialized. We use it to delay 1121 // initial update or build of index. 1122 TimeStamp mStartTime MOZ_GUARDED_BY(sLock); 1123 // Set to true in PreShutdown(), it is checked on variaous places to prevent 1124 // starting any process (write, update, etc.) during shutdown. 1125 bool mShuttingDown MOZ_GUARDED_BY(sLock){false}; 1126 // When set to true, update process should start as soon as possible. This 1127 // flag is set whenever we find some inconsistency which would be fixed by 1128 // update process. The flag is checked always when switching to READY state. 1129 // To make sure we start the update process as soon as possible, methods that 1130 // set this flag should also call StartUpdatingIndexIfNeeded() to cover the 1131 // case when we are currently in READY state. 1132 bool mIndexNeedsUpdate MOZ_GUARDED_BY(sLock){false}; 1133 // Set at the beginning of RemoveAll() which clears the whole index. When 1134 // removing all entries we must stop any pending reading, writing, updating or 1135 // building operation. This flag is checked at various places and it prevents 1136 // we won't start another operation (e.g. canceling reading of the index would 1137 // normally start update or build process) 1138 bool mRemovingAll MOZ_GUARDED_BY(sLock){false}; 1139 // Whether the index file on disk exists and is valid. 1140 bool mIndexOnDiskIsValid MOZ_GUARDED_BY(sLock){false}; 1141 // When something goes wrong during updating or building process, we don't 1142 // mark index clean (and also don't write journal) to ensure that update or 1143 // build will be initiated on the next start. 1144 bool mDontMarkIndexClean MOZ_GUARDED_BY(sLock){false}; 1145 // Timestamp value from index file. It is used during update process to skip 1146 // entries that were last modified before this timestamp. 1147 uint32_t mIndexTimeStamp MOZ_GUARDED_BY(sLock){0}; 1148 // Timestamp of last time the index was dumped to disk. 1149 // NOTE: The index might not be necessarily dumped at this time. The value 1150 // is used to schedule next dump of the index. 1151 TimeStamp mLastDumpTime MOZ_GUARDED_BY(sLock); 1152 1153 // Timer of delayed update/build. 1154 nsCOMPtr<nsITimer> mUpdateTimer MOZ_GUARDED_BY(sLock); 1155 // True when build or update event is posted 1156 bool mUpdateEventPending MOZ_GUARDED_BY(sLock){false}; 1157 1158 // Helper members used when reading/writing index from/to disk. 1159 // Contains number of entries that should be skipped: 1160 // - in hashtable when writing index because they were already written 1161 // - in index file when reading index because they were already read 1162 uint32_t mSkipEntries MOZ_GUARDED_BY(sLock){0}; 1163 // Number of entries that should be written to disk. This is number of entries 1164 // in hashtable that are initialized and are not marked as removed when 1165 // writing begins. 1166 uint32_t mProcessEntries MOZ_GUARDED_BY(sLock){0}; 1167 char* mRWBuf MOZ_GUARDED_BY(sLock){nullptr}; 1168 uint32_t mRWBufSize MOZ_GUARDED_BY(sLock){0}; 1169 uint32_t mRWBufPos MOZ_GUARDED_BY(sLock){0}; 1170 RefPtr<CacheHash> mRWHash MOZ_GUARDED_BY(sLock); 1171 1172 // True if read or write operation is pending. It is used to ensure that 1173 // mRWBuf is not freed until OnDataRead or OnDataWritten is called. 1174 bool mRWPending MOZ_GUARDED_BY(sLock){false}; 1175 1176 // Reading of journal succeeded if true. 1177 bool mJournalReadSuccessfully MOZ_GUARDED_BY(sLock){false}; 1178 1179 // Handle used for writing and reading index file. 1180 RefPtr<CacheFileHandle> mIndexHandle MOZ_GUARDED_BY(sLock); 1181 // Handle used for reading journal file. 1182 RefPtr<CacheFileHandle> mJournalHandle MOZ_GUARDED_BY(sLock); 1183 // Used to check the existence of the file during reading process. 1184 RefPtr<CacheFileHandle> mTmpHandle MOZ_GUARDED_BY(sLock); 1185 1186 RefPtr<FileOpenHelper> mIndexFileOpener MOZ_GUARDED_BY(sLock); 1187 RefPtr<FileOpenHelper> mJournalFileOpener MOZ_GUARDED_BY(sLock); 1188 RefPtr<FileOpenHelper> mTmpFileOpener MOZ_GUARDED_BY(sLock); 1189 1190 // Directory enumerator used when building and updating index. 1191 nsCOMPtr<nsIDirectoryEnumerator> mDirEnumerator MOZ_GUARDED_BY(sLock); 1192 1193 // Main index hashtable. 1194 nsTHashtable<CacheIndexEntry> mIndex MOZ_GUARDED_BY(sLock); 1195 1196 // We cannot add, remove or change any entry in mIndex in states READING and 1197 // WRITING. We track all changes in mPendingUpdates during these states. 1198 nsTHashtable<CacheIndexEntryUpdate> mPendingUpdates MOZ_GUARDED_BY(sLock); 1199 1200 // Contains information statistics for mIndex + mPendingUpdates. 1201 CacheIndexStats mIndexStats MOZ_GUARDED_BY(sLock); 1202 1203 // When reading journal, we must first parse the whole file and apply the 1204 // changes iff the journal was read successfully. mTmpJournal is used to store 1205 // entries from the journal file. We throw away all these entries if parsing 1206 // of the journal fails or the hash does not match. 1207 nsTHashtable<CacheIndexEntry> mTmpJournal MOZ_GUARDED_BY(sLock); 1208 1209 // FrecencyStorage maintains records for eviction. Due to the massive amount 1210 // of calls at startup to the "Contains" method, 1211 // we keep the records in a hashtable for faster lookup. Sorting is always 1212 // done during eviction. This allows us to keep the hashtable fast and 1213 // efficient, while still being able to evict entries based on frecency. 1214 class FrecencyStorage final { 1215 FRIEND_TEST(FrecencyStorageTest, AppendRemoveRecordTest); 1216 FRIEND_TEST(FrecencyStorageTest, ReplaceRecordTest); 1217 FRIEND_TEST(FrecencyStorageTest, ClearTest); 1218 FRIEND_TEST(FrecencyStorageTest, GetSortedSnapshotForEvictionTest); 1219 FRIEND_TEST(FrecencyStorageTest, PerformanceTest); 1220 1221 public: 1222 FrecencyStorage() = default; 1223 1224 // Methods used by CacheIndexEntryAutoManage to keep the storage up to date. 1225 void AppendRecord(CacheIndexRecordWrapper* aRecord, 1226 const StaticMutexAutoLock& aProofOfLock); 1227 1228 void RemoveRecord(CacheIndexRecordWrapper* aRecord, 1229 const StaticMutexAutoLock& aProofOfLock); 1230 1231 void ReplaceRecord(CacheIndexRecordWrapper* aOldRecord, 1232 CacheIndexRecordWrapper* aNewRecord, 1233 const StaticMutexAutoLock& aProofOfLock); 1234 1235 bool RecordExistedUnlocked(CacheIndexRecordWrapper* aRecord); 1236 1237 // EvictionSortedSnapshot is used to transform FrecencyStorage into a sorted 1238 // array which is then used for eviction. 1239 EvictionSortedSnapshot GetSortedSnapshotForEviction(); 1240 1241 size_t Length() const { return mRecs.Count(); } 1242 1243 void Clear(const StaticMutexAutoLock& aProofOfLock) { mRecs.Clear(); } 1244 1245 private: 1246 friend class CacheIndex; 1247 nsTHashtable<nsRefPtrHashKey<CacheIndexRecordWrapper>> mRecs; 1248 }; 1249 1250 FrecencyStorage mFrecencyStorage MOZ_GUARDED_BY(sLock); 1251 1252 nsTArray<CacheIndexIterator*> mIterators MOZ_GUARDED_BY(sLock); 1253 1254 // This flag is true iff we are between CacheStorageService:Clear() and 1255 // processing all contexts to be evicted. It will make UI to show 1256 // "calculating" instead of any intermediate cache size. 1257 bool mAsyncGetDiskConsumptionBlocked MOZ_GUARDED_BY(sLock){false}; 1258 1259 class DiskConsumptionObserver : public Runnable { 1260 public: 1261 static DiskConsumptionObserver* Init( 1262 nsICacheStorageConsumptionObserver* aObserver) { 1263 nsWeakPtr observer = do_GetWeakReference(aObserver); 1264 if (!observer) return nullptr; 1265 1266 return new DiskConsumptionObserver(observer); 1267 } 1268 1269 void OnDiskConsumption(int64_t aSize) { 1270 mSize = aSize; 1271 NS_DispatchToMainThread(this); 1272 } 1273 1274 private: 1275 explicit DiskConsumptionObserver(nsWeakPtr const& aWeakObserver) 1276 : Runnable("net::CacheIndex::DiskConsumptionObserver"), 1277 mObserver(aWeakObserver), 1278 mSize(0) {} 1279 virtual ~DiskConsumptionObserver() { 1280 if (mObserver && !NS_IsMainThread()) { 1281 NS_ReleaseOnMainThread("DiskConsumptionObserver::mObserver", 1282 mObserver.forget()); 1283 } 1284 } 1285 1286 NS_IMETHOD Run() override { 1287 MOZ_ASSERT(NS_IsMainThread()); 1288 1289 nsCOMPtr<nsICacheStorageConsumptionObserver> observer = 1290 do_QueryReferent(mObserver); 1291 1292 mObserver = nullptr; 1293 1294 if (observer) { 1295 observer->OnNetworkCacheDiskConsumption(mSize); 1296 } 1297 1298 return NS_OK; 1299 } 1300 1301 nsWeakPtr mObserver; 1302 int64_t mSize; 1303 }; 1304 1305 // List of async observers that want to get disk consumption information 1306 nsTArray<RefPtr<DiskConsumptionObserver>> mDiskConsumptionObservers 1307 MOZ_GUARDED_BY(sLock); 1308 1309 // Number of bytes written to the cache since the last telemetry report 1310 uint64_t mTotalBytesWritten MOZ_GUARDED_BY(sLock){0}; 1311 }; 1312 1313 } // namespace net 1314 } // namespace mozilla 1315 1316 #endif