CacheFileIOManager.h (20894B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #ifndef CacheFileIOManager__h__ 6 #define CacheFileIOManager__h__ 7 8 #include "CacheIOThread.h" 9 #include "CacheStorageService.h" 10 #include "CacheHashUtils.h" 11 #include "nsIEventTarget.h" 12 #include "nsINamed.h" 13 #include "nsITimer.h" 14 #include "nsCOMPtr.h" 15 #include "mozilla/Atomics.h" 16 #include "mozilla/SHA1.h" 17 #include "mozilla/StaticPtr.h" 18 #include "mozilla/TimeStamp.h" 19 #include "nsTArray.h" 20 #include "nsString.h" 21 #include "nsTHashtable.h" 22 #include "prio.h" 23 #include "Dictionary.h" 24 25 // #define DEBUG_HANDLES 1 26 #if !defined(MOZ_WIDGET_ANDROID) 27 # define MOZ_CACHE_ASYNC_IO 1 28 #endif 29 30 class nsIFile; 31 class nsITimer; 32 class nsIDirectoryEnumerator; 33 class nsILoadContextInfo; 34 class nsIRunnable; 35 36 namespace mozilla { 37 namespace net { 38 39 class CacheFile; 40 class CacheFileIOListener; 41 class PendingItemComparator; 42 43 #ifdef DEBUG_HANDLES 44 class CacheFileHandlesEntry; 45 #endif 46 47 #define ENTRIES_DIR "entries" 48 #define DOOMED_DIR "doomed" 49 #define TRASH_DIR "trash" 50 51 class CacheFileHandle final : public nsISupports { 52 public: 53 enum class PinningStatus : uint32_t { UNKNOWN, NON_PINNED, PINNED }; 54 55 NS_DECL_THREADSAFE_ISUPPORTS 56 bool DispatchRelease(); 57 58 CacheFileHandle(const SHA1Sum::Hash* aHash, bool aPriority, 59 PinningStatus aPinning); 60 CacheFileHandle(const nsACString& aKey, bool aPriority, 61 PinningStatus aPinning); 62 void Log(); 63 bool IsDoomed() const { return mIsDoomed; } 64 const SHA1Sum::Hash* Hash() const { return mHash; } 65 int64_t FileSize() const { return mFileSize; } 66 uint32_t FileSizeInK() const; 67 bool IsPriority() const { return mPriority; } 68 bool FileExists() const { return mFileExists; } 69 bool IsClosed() const { return mClosed; } 70 bool IsSpecialFile() const { return mSpecialFile; } 71 nsCString& Key() { return mKey; } 72 73 // Returns false when this handle has been doomed based on the pinning state 74 // update. 75 bool SetPinned(bool aPinned); 76 void SetInvalid() { mInvalid = true; } 77 78 // Memory reporting 79 size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; 80 size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const; 81 82 #if defined(MOZ_CACHE_ASYNC_IO) 83 void StartAsyncOperation(); 84 void EndAsyncOperation(); 85 bool IsAsyncOperationRunning() const { return mAsyncRunning > 0; } 86 87 bool WaitForAsyncCompletion(nsIRunnable* aEvent, uint32_t aLevel); 88 #endif 89 90 private: 91 friend class CacheFileIOManager; 92 friend class CacheFileHandles; 93 friend class ReleaseNSPRHandleEvent; 94 friend class PendingItemComparator; 95 96 virtual ~CacheFileHandle(); 97 98 const SHA1Sum::Hash* mHash; 99 mozilla::Atomic<bool, ReleaseAcquire> mIsDoomed; 100 mozilla::Atomic<bool, ReleaseAcquire> mClosed; 101 102 // mPriority and mSpecialFile are plain "bool", not "bool:1", so as to 103 // avoid bitfield races with the byte containing mInvalid et al. See 104 // bug 1278502. 105 bool const mPriority; 106 bool const mSpecialFile; 107 108 mozilla::Atomic<bool, Relaxed> mInvalid; 109 110 // These bit flags are all accessed only on the IO thread 111 bool mFileExists : 1; // This means that the file should exists, 112 // but it can be still deleted by OS/user 113 // and then a subsequent OpenNSPRFileDesc() 114 // will fail. 115 116 // Both initially false. Can be raised to true only when this handle is to be 117 // doomed during the period when the pinning status is unknown. After the 118 // pinning status determination we check these flags and possibly doom. These 119 // flags are only accessed on the IO thread. 120 bool mDoomWhenFoundPinned : 1; 121 bool mDoomWhenFoundNonPinned : 1; 122 // Set when after shutdown AND: 123 // - when writing: writing data (not metadata) OR the physical file handle is 124 // not currently open 125 // - when truncating: the physical file handle is not currently open 126 // When set it prevents any further writes or truncates on such handles to 127 // happen immediately after shutdown and gives a chance to write metadata of 128 // already open files quickly as possible (only that renders them actually 129 // usable by the cache.) 130 bool mKilled : 1; 131 #if defined(MOZ_CACHE_ASYNC_IO) 132 // There is a async operation running on this handle. 133 uint32_t mAsyncRunning{0}; 134 #endif 135 // For existing files this is always pre-set to UNKNOWN. The status is 136 // udpated accordingly after the matadata has been parsed. For new files the 137 // flag is set according to which storage kind is opening the cache entry and 138 // remains so for the handle's lifetime. The status can only change from 139 // UNKNOWN (if set so initially) to one of PINNED or NON_PINNED and it stays 140 // unchanged afterwards. This status is only accessed on the IO thread. 141 PinningStatus mPinning; 142 143 nsCOMPtr<nsIFile> mFile; 144 Atomic<int64_t, Relaxed> mFileSize; 145 PRFileDesc* mFD; // if null then the file doesn't exists on the disk 146 nsCString mKey; 147 #if defined(MOZ_CACHE_ASYNC_IO) 148 using PendingItem = std::pair<RefPtr<nsIRunnable>, uint32_t>; 149 // Events that are being blocked by an async operation running on this handle. 150 nsTArray<PendingItem> mPendingEvents; 151 #endif 152 }; 153 154 class CacheFileHandles { 155 public: 156 CacheFileHandles(); 157 ~CacheFileHandles(); 158 159 nsresult GetHandle(const SHA1Sum::Hash* aHash, CacheFileHandle** _retval); 160 already_AddRefed<CacheFileHandle> NewHandle(const SHA1Sum::Hash*, 161 bool aPriority, 162 CacheFileHandle::PinningStatus); 163 void RemoveHandle(CacheFileHandle* aHandle); 164 void GetAllHandles(nsTArray<RefPtr<CacheFileHandle> >* _retval); 165 void GetActiveHandles(nsTArray<RefPtr<CacheFileHandle> >* _retval); 166 void ClearAll(); 167 uint32_t HandleCount(); 168 169 #ifdef DEBUG_HANDLES 170 void Log(CacheFileHandlesEntry* entry); 171 #endif 172 173 // Memory reporting 174 size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; 175 size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const; 176 177 class HandleHashKey : public PLDHashEntryHdr { 178 public: 179 using KeyType = const SHA1Sum::Hash&; 180 using KeyTypePointer = const SHA1Sum::Hash*; 181 182 explicit HandleHashKey(KeyTypePointer aKey) { 183 MOZ_COUNT_CTOR(HandleHashKey); 184 mHash = MakeUnique<uint8_t[]>(SHA1Sum::kHashSize); 185 memcpy(mHash.get(), aKey, sizeof(SHA1Sum::Hash)); 186 } 187 HandleHashKey(const HandleHashKey& aOther) { 188 MOZ_ASSERT_UNREACHABLE("HandleHashKey copy constructor is forbidden!"); 189 } 190 MOZ_COUNTED_DTOR(HandleHashKey) 191 192 bool KeyEquals(KeyTypePointer aKey) const { 193 return memcmp(mHash.get(), aKey, sizeof(SHA1Sum::Hash)) == 0; 194 } 195 static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; } 196 static PLDHashNumber HashKey(KeyTypePointer aKey) { 197 return (reinterpret_cast<const uint32_t*>(aKey))[0]; 198 } 199 200 void AddHandle(CacheFileHandle* aHandle); 201 void RemoveHandle(CacheFileHandle* aHandle); 202 already_AddRefed<CacheFileHandle> GetNewestHandle(); 203 void GetHandles(nsTArray<RefPtr<CacheFileHandle> >& aResult); 204 205 SHA1Sum::Hash* Hash() const { 206 return reinterpret_cast<SHA1Sum::Hash*>(mHash.get()); 207 } 208 bool IsEmpty() const { return mHandles.Length() == 0; } 209 210 enum { ALLOW_MEMMOVE = true }; 211 212 #ifdef DEBUG 213 void AssertHandlesState(); 214 #endif 215 216 // Memory reporting 217 size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; 218 size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const; 219 220 private: 221 // We can't make this UniquePtr<SHA1Sum::Hash>, because you can't have 222 // UniquePtrs with known bounds. So we settle for this representation 223 // and using appropriate casts when we need to access it as a 224 // SHA1Sum::Hash. 225 UniquePtr<uint8_t[]> mHash; 226 // Use weak pointers since the hash table access is on a single thread 227 // only and CacheFileHandle removes itself from this table in its dtor 228 // that may only be called on the same thread as we work with the hashtable 229 // since we dispatch its Release() to this thread. 230 nsTArray<CacheFileHandle*> mHandles; 231 }; 232 233 private: 234 nsTHashtable<HandleHashKey> mTable; 235 }; 236 237 //////////////////////////////////////////////////////////////////////////////// 238 239 class OpenFileEvent; 240 class ReadEvent; 241 class WriteEvent; 242 class MetadataWriteScheduleEvent; 243 class CacheFileContextEvictor; 244 245 #define CACHEFILEIOLISTENER_IID \ 246 {/* dcaf2ddc-17cf-4242-bca1-8c86936375a5 */ \ 247 0xdcaf2ddc, \ 248 0x17cf, \ 249 0x4242, \ 250 {0xbc, 0xa1, 0x8c, 0x86, 0x93, 0x63, 0x75, 0xa5}} 251 252 class CacheFileIOListener : public nsISupports { 253 public: 254 NS_INLINE_DECL_STATIC_IID(CACHEFILEIOLISTENER_IID) 255 256 NS_IMETHOD OnFileOpened(CacheFileHandle* aHandle, nsresult aResult) = 0; 257 NS_IMETHOD OnDataWritten(CacheFileHandle* aHandle, const char* aBuf, 258 nsresult aResult) = 0; 259 NS_IMETHOD OnDataRead(CacheFileHandle* aHandle, char* aBuf, 260 nsresult aResult) = 0; 261 NS_IMETHOD OnFileDoomed(CacheFileHandle* aHandle, nsresult aResult) = 0; 262 NS_IMETHOD OnEOFSet(CacheFileHandle* aHandle, nsresult aResult) = 0; 263 NS_IMETHOD OnFileRenamed(CacheFileHandle* aHandle, nsresult aResult) = 0; 264 265 virtual bool IsKilled() { return false; } 266 }; 267 268 class CacheFileIOManager final : public nsITimerCallback, public nsINamed { 269 public: 270 NS_DECL_THREADSAFE_ISUPPORTS 271 NS_DECL_NSITIMERCALLBACK 272 NS_DECL_NSINAMED 273 274 enum { 275 OPEN = 0U, 276 CREATE = 1U, 277 CREATE_NEW = 2U, 278 PRIORITY = 4U, 279 SPECIAL_FILE = 8U, 280 PINNED = 16U 281 }; 282 283 CacheFileIOManager(); 284 285 static nsresult Init(); 286 static nsresult Shutdown(); 287 static nsresult OnProfile(); 288 static nsresult OnDelayedStartupFinished(); 289 static nsresult OnIdleDaily(); 290 static already_AddRefed<nsIEventTarget> IOTarget(); 291 static already_AddRefed<CacheIOThread> IOThread(); 292 static bool IsOnIOThread(); 293 static bool IsOnIOThreadOrCeased(); 294 static bool IsShutdown(); 295 296 // Make aFile's WriteMetadataIfNeeded be called automatically after 297 // a short interval. 298 static nsresult ScheduleMetadataWrite(CacheFile* aFile); 299 // Remove aFile from the scheduling registry array. 300 // WriteMetadataIfNeeded will not be automatically called. 301 static nsresult UnscheduleMetadataWrite(CacheFile* aFile); 302 // Shuts the scheduling off and flushes all pending metadata writes. 303 static nsresult ShutdownMetadataWriteScheduling(); 304 305 static nsresult OpenFile(const nsACString& aKey, uint32_t aFlags, 306 CacheFileIOListener* aCallback); 307 static nsresult Read(CacheFileHandle* aHandle, int64_t aOffset, char* aBuf, 308 int32_t aCount, CacheFileIOListener* aCallback); 309 // This function must be called with a callback. The caller is responsible for 310 // releasing |aBuf|. 311 static nsresult Write(CacheFileHandle* aHandle, int64_t aOffset, 312 const char* aBuf, int32_t aCount, bool aValidate, 313 bool aTruncate, CacheFileIOListener* aCallback); 314 // Similar to the above, but without the callback. Note that |aBuf| will be 315 // released by CacheFileIOManager. 316 static nsresult WriteWithoutCallback(CacheFileHandle* aHandle, 317 int64_t aOffset, char* aBuf, 318 int32_t aCount, bool aValidate, 319 bool aTruncate); 320 // PinningDoomRestriction: 321 // NO_RESTRICTION 322 // no restriction is checked, the file is simply always doomed 323 // DOOM_WHEN_(NON)_PINNED, we branch based on the pinning status of the 324 // handle: 325 // UNKNOWN: the handle is marked to be doomed when later found (non)pinned 326 // PINNED/NON_PINNED: doom only when the restriction matches the pin status 327 // and the handle has not yet been required to doom during the UNKNOWN 328 // period 329 enum PinningDoomRestriction { 330 NO_RESTRICTION, 331 DOOM_WHEN_NON_PINNED, 332 DOOM_WHEN_PINNED 333 }; 334 static nsresult DoomFile(CacheFileHandle* aHandle, 335 CacheFileIOListener* aCallback); 336 static nsresult DoomFileByKey(const nsACString& aKey, 337 CacheFileIOListener* aCallback); 338 static nsresult ReleaseNSPRHandle(CacheFileHandle* aHandle); 339 static nsresult TruncateSeekSetEOF(CacheFileHandle* aHandle, 340 int64_t aTruncatePos, int64_t aEOFPos, 341 CacheFileIOListener* aCallback); 342 static nsresult RenameFile(CacheFileHandle* aHandle, 343 const nsACString& aNewName, 344 CacheFileIOListener* aCallback); 345 static nsresult EvictIfOverLimit(); 346 static nsresult EvictAll(); 347 static nsresult EvictByContext(nsILoadContextInfo* aLoadContextInfo, 348 bool aPinned, const nsAString& aOrigin, 349 const nsAString& aBaseDomain = u""_ns); 350 351 static nsresult InitIndexEntry(CacheFileHandle* aHandle, 352 OriginAttrsHash aOriginAttrsHash, 353 bool aAnonymous, bool aPinning); 354 static nsresult UpdateIndexEntry(CacheFileHandle* aHandle, 355 const uint32_t* aFrecency, 356 const bool* aHasAltData, 357 const uint16_t* aOnStartTime, 358 const uint16_t* aOnStopTime, 359 const uint8_t* aContentType); 360 361 static nsresult UpdateIndexEntry(); 362 363 enum EEnumerateMode { ENTRIES, DOOMED }; 364 365 static void GetCacheDirectory(nsIFile** result); 366 #if defined(MOZ_WIDGET_ANDROID) 367 static void GetProfilelessCacheDirectory(nsIFile** result); 368 #endif 369 370 // Calls synchronously OnEntryInfo for an entry with the given hash. 371 // Tries to find an existing entry in the service hashtables first, if not 372 // found, loads synchronously from disk file. 373 // Callable on the IO thread only. 374 static nsresult GetEntryInfo( 375 const SHA1Sum::Hash* aHash, 376 CacheStorageService::EntryInfoCallback* aCallback); 377 378 // Memory reporting 379 static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf); 380 static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf); 381 382 private: 383 friend class CacheFileHandle; 384 friend class CacheFileChunk; 385 friend class CacheFile; 386 friend class ShutdownEvent; 387 friend class OpenFileEvent; 388 friend class CloseHandleEvent; 389 friend class ReadEvent; 390 friend class WriteEvent; 391 friend class DoomFileEvent; 392 friend class DoomFileByKeyEvent; 393 friend class ReleaseNSPRHandleEvent; 394 friend class TruncateSeekSetEOFEvent; 395 friend class RenameFileEvent; 396 friend class CacheIndex; 397 friend class MetadataWriteScheduleEvent; 398 friend class CacheFileContextEvictor; 399 400 virtual ~CacheFileIOManager(); 401 402 nsresult InitInternal(); 403 void ShutdownInternal(); 404 405 nsresult OpenFileInternal(const SHA1Sum::Hash* aHash, const nsACString& aKey, 406 uint32_t aFlags, CacheFileHandle** _retval); 407 nsresult OpenSpecialFileInternal(const nsACString& aKey, uint32_t aFlags, 408 CacheFileHandle** _retval); 409 void CloseHandleInternal(CacheFileHandle* aHandle); 410 nsresult ReadInternal(CacheFileHandle* aHandle, int64_t aOffset, char* aBuf, 411 int32_t aCount, ReadEvent* aReadEvent); 412 nsresult WriteInternal(CacheFileHandle* aHandle, int64_t aOffset, 413 const char* aBuf, int32_t aCount, bool aValidate, 414 bool aTruncate); 415 nsresult DoomFileInternal( 416 CacheFileHandle* aHandle, 417 PinningDoomRestriction aPinningDoomRestriction = NO_RESTRICTION, 418 bool aClearDirectory = true); 419 nsresult DoomFileByKeyInternal(const SHA1Sum::Hash* aHash); 420 nsresult MaybeReleaseNSPRHandleInternal(CacheFileHandle* aHandle, 421 bool aIgnoreShutdownLag = false); 422 nsresult TruncateSeekSetEOFInternal(CacheFileHandle* aHandle, 423 int64_t aTruncatePos, int64_t aEOFPos); 424 nsresult RenameFileInternal(CacheFileHandle* aHandle, 425 const nsACString& aNewName); 426 nsresult EvictIfOverLimitInternal(); 427 nsresult OverLimitEvictionInternal(); 428 nsresult EvictAllInternal(); 429 nsresult EvictByContextInternal(nsILoadContextInfo* aLoadContextInfo, 430 bool aPinned, const nsAString& aOrigin, 431 const nsAString& aBaseDomain = u""_ns); 432 433 nsresult TrashDirectory(nsIFile* aFile); 434 static void OnTrashTimer(nsITimer* aTimer, void* aClosure); 435 nsresult StartRemovingTrash(); 436 nsresult RemoveTrashInternal(); 437 nsresult FindTrashDirToRemove(); 438 439 nsresult CreateFile(CacheFileHandle* aHandle); 440 static void HashToStr(const SHA1Sum::Hash* aHash, nsACString& _retval); 441 static nsresult StrToHash(const nsACString& aHash, SHA1Sum::Hash* _retval); 442 nsresult GetFile(const SHA1Sum::Hash* aHash, nsIFile** _retval); 443 nsresult GetSpecialFile(const nsACString& aKey, nsIFile** _retval); 444 nsresult GetDoomedFile(nsIFile** _retval); 445 nsresult IsEmptyDirectory(nsIFile* aFile, bool* _retval); 446 nsresult CheckAndCreateDir(nsIFile* aFile, const char* aDir, 447 bool aEnsureEmptyDir); 448 nsresult CreateCacheTree(); 449 nsresult OpenNSPRHandle(CacheFileHandle* aHandle, bool aCreate = false); 450 void NSPRHandleUsed(CacheFileHandle* aHandle); 451 452 // Removing all cache files during shutdown 453 nsresult SyncRemoveDir(nsIFile* aFile, const char* aDir); 454 void SyncRemoveAllCacheFiles(); 455 456 nsresult ScheduleMetadataWriteInternal(CacheFile* aFile); 457 void UnscheduleMetadataWriteInternal(CacheFile* aFile); 458 void ShutdownMetadataWriteSchedulingInternal(); 459 460 static nsresult CacheIndexStateChanged(); 461 void CacheIndexStateChangedInternal(); 462 463 // Dispatches a purgeHTTP background task to delete the cache directoy 464 // indicated by aCacheDirName. 465 // When this feature is enabled, a task will be dispatched at shutdown 466 // or after browser startup (to cleanup potential left-over directories) 467 nsresult DispatchPurgeTask(const nsCString& aCacheDirName, 468 const nsCString& aSecondsToWait, 469 const nsCString& aPurgeExtension); 470 471 #if defined(MOZ_CACHE_ASYNC_IO) 472 void DispatchPendingEvents(); 473 #endif 474 475 // Smart size calculation. UpdateSmartCacheSize() must be called on IO thread. 476 // It is called in EvictIfOverLimitInternal() just before we decide whether to 477 // start overlimit eviction or not and also in OverLimitEvictionInternal() 478 // before we start an eviction loop. 479 nsresult UpdateSmartCacheSize(int64_t aFreeSpace); 480 481 // Memory reporting (private part) 482 size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const; 483 484 static StaticRefPtr<CacheFileIOManager> gInstance; 485 486 // Pointer to DictionaryCache singleton 487 RefPtr<DictionaryCache> mDictionaryCache; 488 489 TimeStamp mStartTime; 490 // Set true on the IO thread, CLOSE level as part of the internal shutdown 491 // procedure. 492 bool mShuttingDown{false}; 493 RefPtr<CacheIOThread> mIOThread; 494 nsCOMPtr<nsIFile> mCacheDirectory; 495 #if defined(MOZ_WIDGET_ANDROID) 496 // On Android we add the active profile directory name between the path 497 // and the 'cache2' leaf name. However, to delete any leftover data from 498 // times before we were doing it, we still need to access the directory 499 // w/o the profile name in the path. Here it is stored. 500 nsCOMPtr<nsIFile> mCacheProfilelessDirectory; 501 #endif 502 bool mTreeCreated{false}; 503 bool mTreeCreationFailed{false}; 504 CacheFileHandles mHandles; 505 nsTArray<CacheFileHandle*> mHandlesByLastUsed; 506 nsTArray<CacheFileHandle*> mSpecialHandles; 507 nsTArray<RefPtr<CacheFile> > mScheduledMetadataWrites; 508 nsCOMPtr<nsITimer> mMetadataWritesTimer; 509 bool mOverLimitEvicting{false}; 510 // When overlimit eviction is too slow and cache size reaches 105% of the 511 // limit, this flag is set and no other content is cached to prevent 512 // uncontrolled cache growing. 513 bool mCacheSizeOnHardLimit{false}; 514 bool mRemovingTrashDirs{false}; 515 nsCOMPtr<nsITimer> mTrashTimer; 516 nsCOMPtr<nsIFile> mTrashDir; 517 nsCOMPtr<nsIDirectoryEnumerator> mTrashDirEnumerator; 518 nsTArray<nsCString> mFailedTrashDirs; 519 RefPtr<CacheFileContextEvictor> mContextEvictor; 520 TimeStamp mLastSmartSizeTime; 521 #if defined(MOZ_CACHE_ASYNC_IO) 522 using PendingItem = std::pair<RefPtr<nsIRunnable>, uint32_t>; 523 nsTArray<PendingItem> mPendingEvents; 524 // This is used to track how many async operations are running. It 525 // is used to prevent the IO thread from shutting down while there 526 // are still some async operations running. 527 uint32_t mAsyncRunning{0}; 528 #endif 529 }; 530 531 } // namespace net 532 } // namespace mozilla 533 534 #endif