commit 0b4eb92e4ae013da6bb3080509c8c45416343552 parent 2121596c39e539dbf6e882879103e680ad7a7369 Author: Randell Jesup <rjesup@mozilla.com> Date: Tue, 7 Oct 2025 17:55:57 +0000 Bug 1978494: Read and write of per-origin compression dictionary cache entries/metadata r=necko-reviewers,valentin Differential Revision: https://phabricator.services.mozilla.com/D258162 Diffstat:
19 files changed, 789 insertions(+), 273 deletions(-)
diff --git a/netwerk/cache2/CacheFile.cpp b/netwerk/cache2/CacheFile.cpp @@ -2123,7 +2123,8 @@ void CacheFile::RemoveOutput(CacheFileOutputStream* aOutput, nsresult aStatus) { // This is to finalize the Hash calculation if (mDict) { - mDict->FinishFile(); + mDict->FinishHash(); + mDict = nullptr; } mOutput = nullptr; // XXX should this be after NotifyCloseListener? diff --git a/netwerk/cache2/Dictionary.cpp b/netwerk/cache2/Dictionary.cpp @@ -24,7 +24,6 @@ #include "nsILoadContextInfo.h" #include "nsILoadGroup.h" #include "nsIObserverService.h" -#include "nsITimer.h" #include "nsIURI.h" #include "nsInputStreamPump.h" #include "nsNetUtil.h" @@ -70,6 +69,8 @@ LazyLogModule gDictionaryLog("CompressionDictionaries"); StaticRefPtr<DictionaryCache> gDictionaryCache; nsCOMPtr<nsICacheStorage> DictionaryCache::sCacheStorage; +DictionaryCacheEntry::DictionaryCacheEntry(const char* aKey) { mURI = aKey; } + DictionaryCacheEntry::DictionaryCacheEntry(const nsACString& aURI, const nsACString& aPattern, const nsACString& aId, @@ -86,7 +87,10 @@ NS_IMPL_ISUPPORTS(DictionaryCacheEntry, nsICacheEntryOpenCallback, // returns true if the pattern for the dictionary matches the path given bool DictionaryCacheEntry::Match(const nsACString& aFilePath, uint32_t& aLongest) { - MOZ_ASSERT(NS_IsMainThread()); + if (mHash.IsEmpty()) { + // We don't have the file yet + return false; + } // Not worth checking if we wouldn't use it // XXX Check match-dest if (mPattern.Length() > aLongest) { @@ -147,7 +151,7 @@ bool DictionaryCacheEntry::Prefetch(nsILoadContextInfo* aLoadContextInfo, // Start reading the cache entry into memory and call completion // function when done if (mWaitingPrefetch.IsEmpty()) { - if (mDictionaryData.empty()) { + if (!mDictionaryDataComplete) { // We haven't requested it yet from the Cache and don't have it in memory // already // We can't use sCacheStorage because we need the correct LoadContextInfo @@ -166,8 +170,8 @@ bool DictionaryCacheEntry::Prefetch(nsILoadContextInfo* aLoadContextInfo, mWaitingPrefetch.AppendElement(aFunc); cacheStorage->AsyncOpenURIString(mURI, ""_ns, nsICacheStorage::OPEN_READONLY, this); - DICTIONARY_LOG( - ("Started Prefetch for %s", PromiseFlatCString(mURI).get())); + DICTIONARY_LOG(("Started Prefetch for %s, anonymous=%d", mURI.get(), + aLoadContextInfo->IsAnonymous())); return true; } DICTIONARY_LOG(("Prefetch for %s - already have data in memory (%u users)", @@ -197,6 +201,7 @@ void DictionaryCacheEntry::AccumulateHash(const char* aBuf, int32_t aCount) { return; // XXX } if (!mCrypto) { + DICTIONARY_LOG(("Calculating new hash for %s", mURI.get())); // If mCrypto is null, and mDictionaryData is set, we've already got the // data for this dictionary. nsresult rv; @@ -212,29 +217,130 @@ void DictionaryCacheEntry::AccumulateHash(const char* aBuf, int32_t aCount) { mDictionaryData.length())); } -void DictionaryCacheEntry::AccumulateFile(const char* aBuf, int32_t aCount) { - AccumulateHash(aBuf, aCount); // error? - Unused << mDictionaryData.append(aBuf, aCount); - DICTIONARY_LOG(("Accumulate %p: %d bytes, total %zu", this, aCount, - mDictionaryData.length())); -} - -void DictionaryCacheEntry::FinishFile() { +void DictionaryCacheEntry::FinishHash() { MOZ_ASSERT(NS_IsMainThread()); if (mCrypto) { - DICTIONARY_LOG(("Hash was %s", mHash.get())); mCrypto->Finish(true, mHash); - DICTIONARY_LOG(("Set dictionary hash for %p to %s", this, mHash.get())); mCrypto = nullptr; + DICTIONARY_LOG(("Hash for %p (%s) is %s", this, mURI.get(), mHash.get())); + if (mOrigin) { + DICTIONARY_LOG(("Write on hash")); + // This will also move us from mPendingEntries to mEntries + mOrigin->Write(this); + if (!mBlocked) { + mOrigin->FinishAddEntry(this); + } + mOrigin = nullptr; + } } - mDictionaryDataComplete = true; - DICTIONARY_LOG(("Unsuspending %zu channels, Dictionary len %zu", - mWaitingPrefetch.Length(), mDictionaryData.length())); - // if we suspended, un-suspend the channel(s) - for (auto& lambda : mWaitingPrefetch) { - (lambda)(); +} + +// Metadata format: +// * XXX What key should these be under? +// * uint16_t version +// * uint16_t flags ? +// +// * Entries: +// ** CString: URI -- the key +// ** CString: Hash +// ** CString: Pattern +// ** match-dest list, terminated by empty string +// *** CString: Match-dest +// ** CString: Id +// ** CString: type +// We store strings with a delimiter, and use escapes for delimiters or escape +// characters in the source strings. +// + +// Escape the string and append to aOutput +static void EscapeMetadataString(const nsACString& aInput, nsCString& aOutput) { + // First calculate how much we'll need to append. Means we'll walk the source + // twice, but avoids any potential multiple reallocations + const char* src = aInput.BeginReading(); + size_t len = 1; // for initial | + while (*src) { + if (*src == '|' || *src == '\\') { + len += 2; + } else { + len++; + } + src++; + } + aOutput.SetCapacity(aOutput.Length() + len); + src = aInput.BeginReading(); + + aOutput.AppendLiteral("|"); + while (*src) { + if (*src == '|' || *src == '\\') { + aOutput.AppendLiteral("\\"); + } + aOutput.Append(*src++); } - mWaitingPrefetch.Clear(); +} + +void DictionaryCacheEntry::MakeMetadataEntry(nsCString& aNewValue) { + EscapeMetadataString(mHash, aNewValue); + EscapeMetadataString(mPattern, aNewValue); + EscapeMetadataString(mId, aNewValue); + // List of match-dest values is terminated by an empty string + // XXX add match-dest support + EscapeMetadataString(""_ns, aNewValue); + // We don't store type, since we only support type 'raw' We can support + // type in the future by considering missing type as raw without changing the + // format +} + +nsresult DictionaryCacheEntry::Write(nsICacheEntry* aCacheEntry) { + nsAutoCStringN<2048> metadata; + MakeMetadataEntry(metadata); + DICTIONARY_LOG( + ("DictionaryCacheEntry::Write %s %s", mURI.get(), metadata.get())); + return aCacheEntry->SetMetaDataElement(mURI.get(), metadata.get()); +} + +nsresult DictionaryCacheEntry::RemoveEntry(nsICacheEntry* aCacheEntry) { + return aCacheEntry->SetMetaDataElement(mURI.BeginReading(), nullptr); +} + +// Parse - | for field seperator; \ for escape of | or \ . +static const char* GetEncodedString(const char* aSrc, nsACString& aOutput) { + // scan the input string and build the output, handling escapes + aOutput.Truncate(); + while (*aSrc) { + if (*aSrc == '|') { + break; + } + if (*aSrc == '\\') { + aSrc++; + } + aOutput.Append(*aSrc++); + } + return aSrc; +} + +// Parse metadata from DictionaryOrigin +bool DictionaryCacheEntry::ParseMetadata(const char* aSrc) { + MOZ_ASSERT(*aSrc == '|'); + aSrc++; + aSrc = GetEncodedString(aSrc, mHash); + aSrc = GetEncodedString(aSrc, mPattern); + aSrc = GetEncodedString(aSrc, mId); + nsAutoCString temp; + // get match-dest values (list ended with empty string) + do { + aSrc = GetEncodedString(aSrc, temp); + if (!temp.IsEmpty()) { + // XXX store match-dest + } + } while (!temp.IsEmpty()); + // XXX type + aSrc = GetEncodedString(aSrc, temp); + + DICTIONARY_LOG(("Parse entry %s: |%s| %s id=%s", + PromiseFlatCString(mURI).get(), + PromiseFlatCString(mHash).get(), + PromiseFlatCString(mPattern).get(), mId.get())); + return true; } //----------------------------------------------------------------------------- @@ -264,7 +370,9 @@ nsresult DictionaryCacheEntry::ReadCacheData( uint32_t aToOffset, uint32_t aCount, uint32_t* aWriteCount) { DictionaryCacheEntry* self = static_cast<DictionaryCacheEntry*>(aClosure); - self->AccumulateFile(aFromSegment, aCount); + Unused << self->mDictionaryData.append(aFromSegment, aCount); + DICTIONARY_LOG(("Accumulate %p (%s): %d bytes, total %zu", self, + self->mURI.get(), aCount, self->mDictionaryData.length())); *aWriteCount = aCount; return NS_OK; } @@ -273,7 +381,14 @@ NS_IMETHODIMP DictionaryCacheEntry::OnStopRequest(nsIRequest* request, nsresult result) { DICTIONARY_LOG(("DictionaryCacheEntry %s OnStopRequest", mURI.get())); if (NS_SUCCEEDED(result)) { - FinishFile(); + mDictionaryDataComplete = true; + DICTIONARY_LOG(("Unsuspending %zu channels, Dictionary len %zu", + mWaitingPrefetch.Length(), mDictionaryData.length())); + // if we suspended, un-suspend the channel(s) + for (auto& lambda : mWaitingPrefetch) { + (lambda)(); + } + mWaitingPrefetch.Clear(); } else { // XXX // This is problematic - we requested with dcb/dcz, but can't actually @@ -281,9 +396,51 @@ DictionaryCacheEntry::OnStopRequest(nsIRequest* request, nsresult result) { // the entry // XXX } + + // If we're being replaced by a new entry, swap now + RefPtr<DictionaryCacheEntry> self; + if (mReplacement) { + DICTIONARY_LOG(("Replacing entry %p with %p for %s", this, + mReplacement.get(), mURI.get())); + // Make sure we don't destroy ourselves + self = this; + mReplacement->mShouldSuspend = false; + mOrigin->RemoveEntry(this); + // When mReplacement gets all it's data, it will be added to mEntries + mReplacement->UnblockAddEntry(mOrigin); + mOrigin = nullptr; + } + + mStopReceived = true; return NS_OK; } +void DictionaryCacheEntry::UnblockAddEntry(DictionaryOrigin* aOrigin) { + MOZ_ASSERT(NS_IsMainThread()); + if (!mHash.IsEmpty()) { + // Already done, we can move to mEntries now + aOrigin->FinishAddEntry(this); + } + mBlocked = false; +} + +void DictionaryCacheEntry::WriteOnHash(DictionaryOrigin* aOrigin) { + bool hasHash = false; + { + MOZ_ASSERT(NS_IsMainThread()); + if (!mHash.IsEmpty()) { + hasHash = true; + } + } + if (hasHash) { + DICTIONARY_LOG(("Write already hashed")); + aOrigin->Write(this); + } else if (!mStopReceived) { + // This creates a cycle, but we'll clear it when we get FinishFile + mOrigin = aOrigin; + } +} + //----------------------------------------------------------------------------- // nsICacheEntryOpenCallback implementation //----------------------------------------------------------------------------- @@ -302,8 +459,8 @@ DictionaryCacheEntry::OnCacheEntryCheck(nsICacheEntry* aEntry, NS_IMETHODIMP DictionaryCacheEntry::OnCacheEntryAvailable(nsICacheEntry* entry, bool isNew, nsresult status) { - DICTIONARY_LOG(("OnCacheEntryAvailable %s, result %u", - PromiseFlatCString(mURI).get(), (uint32_t)status)); + DICTIONARY_LOG(("OnCacheEntryAvailable %s, result %u, entry %p", mURI.get(), + (uint32_t)status, entry)); if (entry) { nsCOMPtr<nsIInputStream> stream; entry->OpenInputStream(0, getter_AddRefs(stream)); @@ -322,6 +479,13 @@ DictionaryCacheEntry::OnCacheEntryAvailable(nsICacheEntry* entry, bool isNew, return NS_OK; // just ignore } DICTIONARY_LOG(("Waiting for data")); + } else { + // This shouldn't happen.... we should only be fetching something we know + // is in the cache. + // Presumably status is something like NS_ERROR_FILE_NOT_FOUND + // XXX Error out any channels waiting on this cache entry. Also, + // remove the dictionary entry from the origin? + DICTIONARY_LOG(("Prefetched cache entry not available!")); } return NS_OK; @@ -329,30 +493,55 @@ DictionaryCacheEntry::OnCacheEntryAvailable(nsICacheEntry* entry, bool isNew, //---------------------------------------------------------------------------------- -class DictionaryOriginReader final : public nsICacheEntryOpenCallback { +class DictionaryOriginReader final : public nsICacheEntryOpenCallback, + public nsIStreamListener { NS_DECL_THREADSAFE_ISUPPORTS NS_DECL_NSICACHEENTRYOPENCALLBACK + NS_DECL_NSIREQUESTOBSERVER + NS_DECL_NSISTREAMLISTENER DictionaryOriginReader() {} - // Note: don't do this in constructor, since we may call mCallback and - // the release the self-reference - void Init(nsACString& aURI, - const std::function<nsresult(DictionaryCacheEntry*)>& aCallback) { + // if aOrigin is nullptr, we're just reading; if it's set then we're adding + // a Dictionary to a maybe-existing Origin + void Start(DictionaryOrigin* aOrigin, nsACString& aKey, nsIURI* aURI, + DictionaryCache* aCache, + const std::function<nsresult(DictionaryCacheEntry*)>& aCallback) { + mOrigin = aOrigin; + mURI = aURI; mCallback = aCallback; + mCache = aCache; mSelf = this; // keep this alive until we call aCallback + + // The cache entry is for originattribute extension of + // META_DICTIONARY_PREFIX, plus key of prepath + + DICTIONARY_LOG(("DictionaryOriginReader::Start(%s): %p", + PromiseFlatCString(aKey).get(), this)); DictionaryCache::sCacheStorage->AsyncOpenURIString( - aURI, ""_ns, nsICacheStorage::OPEN_READONLY, this); + aKey, META_DICTIONARY_PREFIX, + aOrigin + ? nsICacheStorage::OPEN_NORMALLY + : nsICacheStorage::OPEN_READONLY | nsICacheStorage::OPEN_SECRETLY, + this); } private: ~DictionaryOriginReader() {} + RefPtr<DictionaryOrigin> mOrigin; + nsCOMPtr<nsIURI> mURI; std::function<nsresult(DictionaryCacheEntry*)> mCallback; + RefPtr<DictionaryCache> mCache; RefPtr<DictionaryOriginReader> mSelf; }; -NS_IMPL_ISUPPORTS(DictionaryOriginReader, nsICacheEntryOpenCallback) +NS_IMPL_ISUPPORTS(DictionaryOriginReader, nsICacheEntryOpenCallback, + nsIStreamListener) + +//----------------------------------------------------------------------------- +// nsICacheEntryOpenCallback implementation +//----------------------------------------------------------------------------- NS_IMETHODIMP DictionaryOriginReader::OnCacheEntryCheck(nsICacheEntry* entry, uint32_t* result) { @@ -364,14 +553,81 @@ NS_IMETHODIMP DictionaryOriginReader::OnCacheEntryCheck(nsICacheEntry* entry, } NS_IMETHODIMP DictionaryOriginReader::OnCacheEntryAvailable( - nsICacheEntry* entry, bool isNew, nsresult result) { + nsICacheEntry* aCacheEntry, bool isNew, nsresult result) { MOZ_ASSERT(NS_IsMainThread(), "Got cache entry off main thread!"); DICTIONARY_LOG( - ("Dictionary::OnCacheEntryAvailable this=%p for entry %p", this, entry)); - // XXX + ("DictionaryOriginReader::OnCacheEntryAvailable this=%p for entry %p", + this, aCacheEntry)); - (mCallback)(nullptr); - mSelf = nullptr; // this will delete 'this' + if (!aCacheEntry) { + // Didn't have any dictionaries for this origin + (mCallback)(nullptr); + return NS_OK; + } + + // If we have an mOrigin, then it's already been created and inserted + if (mOrigin) { + mOrigin->SetCacheEntry(aCacheEntry); + // There's no data in the cache entry, just metadata + nsCOMPtr<nsICacheEntryMetaDataVisitor> metadata(mOrigin); + aCacheEntry->VisitMetaData(metadata); + + // We only have mOrigin if we're creating it in AddEntry(), so no + // need to Match() + (mCallback)(nullptr); + mSelf = nullptr; // this will delete 'this' + return NS_OK; + } + + nsCString prepath; + if (NS_FAILED(mURI->GetPrePath(prepath))) { + (mCallback)(nullptr); + return NS_ERROR_FAILURE; + } + // Add the Origin to the hash in DictionaryCache + Unused << mCache->mDictionaryCache.WithEntryHandle( + prepath, [&](auto&& entry) { + auto& origin = entry.OrInsertWith( + [&] { return new DictionaryOrigin(prepath, aCacheEntry); }); + + // There's no data in the cache entry, just metadata + nsCOMPtr<nsICacheEntryMetaDataVisitor> metadata(origin); + aCacheEntry->VisitMetaData(metadata); + + nsCString path; + mURI->GetPathQueryRef(path); + RefPtr<DictionaryCacheEntry> result = origin->Match(path); + + (mCallback)(result); + mSelf = nullptr; // this will delete 'this' + return NS_OK; + }); + return NS_OK; +} + +//----------------------------------------------------------------------------- +// nsIStreamListener implementation +//----------------------------------------------------------------------------- + +NS_IMETHODIMP +DictionaryOriginReader::OnStartRequest(nsIRequest* request) { + DICTIONARY_LOG(("DictionaryOriginReader %p OnStartRequest", this)); + return NS_OK; +} + +NS_IMETHODIMP +DictionaryOriginReader::OnDataAvailable(nsIRequest* request, + nsIInputStream* aInputStream, + uint64_t aOffset, uint32_t aCount) { + uint32_t n; + DICTIONARY_LOG( + ("DictionaryOriginReader %p OnDataAvailable %u", this, aCount)); + return NS_OK; +} + +NS_IMETHODIMP +DictionaryOriginReader::OnStopRequest(nsIRequest* request, nsresult result) { + DICTIONARY_LOG(("DictionaryOriginReader %p OnStopRequest", this)); return NS_OK; } @@ -385,15 +641,17 @@ already_AddRefed<DictionaryCache> DictionaryCache::GetInstance() { } nsresult DictionaryCache::Init() { - nsCOMPtr<nsICacheStorageService> cacheStorageService( - components::CacheStorage::Service()); - if (!cacheStorageService) { - return NS_ERROR_FAILURE; - } - nsresult rv = cacheStorageService->DiskCacheStorage( - nullptr, getter_AddRefs(sCacheStorage)); // Don't need a load context - if (NS_FAILED(rv)) { - return rv; + if (XRE_IsParentProcess()) { + nsCOMPtr<nsICacheStorageService> cacheStorageService( + components::CacheStorage::Service()); + if (!cacheStorageService) { + return NS_ERROR_FAILURE; + } + nsresult rv = cacheStorageService->DiskCacheStorage( + nullptr, getter_AddRefs(sCacheStorage)); // Don't need a load context + if (NS_FAILED(rv)) { + return rv; + } } DICTIONARY_LOG(("Inited DictionaryCache %p", sCacheStorage.get())); return NS_OK; @@ -403,76 +661,61 @@ nsresult DictionaryCache::AddEntry(nsIURI* aURI, const nsACString& aKey, const nsACString& aPattern, const nsACString& aId, const Maybe<nsCString>& aHash, + bool aNewEntry, DictionaryCacheEntry** aDictEntry) { // Note that normally we're getting an entry in and until all the data // has been received, we can't use it. The Hash being null is a flag // that it's not yet valid. - nsCString hostport; - if (NS_FAILED(aURI->GetHostPort(hostport))) { - return NS_ERROR_FAILURE; - } - // create for the origin if it doesn't exist - Unused << mDictionaryCache.WithEntryHandle(hostport, [&](auto&& entry) { - auto& list = entry.OrInsertWith([&] { return new DictCacheList; }); - - for (const auto& dict : *list) { - // We replace on the URI being the same (not id, which is just an - // arbitrary extra id sent back to the server) - if (dict->GetURI().Equals(aKey)) { - // We're overwriting an existing entry, probably with a new hash. It - // might be the same, of course. - // If there are outstanding requests using this entry, we need - // to keep it alive for the users until they're done, but not use - // it for any new requests. Remove the existing entry from the list - // and replace with a new one; existing requests will have a ref to it. - DICTIONARY_LOG(("Replacing dictionary for %s: %p", - PromiseFlatCString(dict->GetURI()).get(), dict)); - dict->remove(); - break; - } - } - // New entry - RefPtr<DictionaryCacheEntry> dict = - new DictionaryCacheEntry(aKey, aPattern, aId, aHash); - DICTIONARY_LOG(("New dictionary for %s: %p", PromiseFlatCString(aKey).get(), - dict.get())); - list->insertFront(dict); + RefPtr<DictionaryCacheEntry> dict = + new DictionaryCacheEntry(aKey, aPattern, aId, aHash); + dict = AddEntry(aURI, aNewEntry, dict); + if (dict) { *aDictEntry = do_AddRef(dict).take(); - - // Queue event to flush dictionary metadata to the cache - // XXX return NS_OK; - }); - return NS_OK; + } + DICTIONARY_LOG( + ("Failed adding entry for %s", PromiseFlatCString(aKey).get())); + *aDictEntry = nullptr; + return NS_ERROR_FAILURE; } -nsresult DictionaryCache::AddEntry(nsIURI* aURI, - DictionaryCacheEntry* aDictEntry) { +already_AddRefed<DictionaryCacheEntry> DictionaryCache::AddEntry( + nsIURI* aURI, bool aNewEntry, DictionaryCacheEntry* aDictEntry) { // Note that normally we're getting an entry in and until all the data // has been received, we can't use it. The Hash being null is a flag // that it's not yet valid. nsCString prepath; if (NS_FAILED(aURI->GetPrePath(prepath))) { - return NS_ERROR_FAILURE; + return nullptr; } // create for the origin if it doesn't exist + RefPtr<DictionaryCacheEntry> newEntry; Unused << mDictionaryCache.WithEntryHandle(prepath, [&](auto&& entry) { - auto& list = entry.OrInsertWith([&] { return new DictCacheList; }); - - // Remove any entry for the same item - for (const auto& dict : *list) { - if (dict->GetURI().Equals(aDictEntry->GetURI())) { - // We're overwriting an existing entry. It might be the same, of - // course - dict->remove(); - return NS_OK; - } - } - - list->insertFront(aDictEntry); + auto& origin = entry.OrInsertWith([&] { + RefPtr<DictionaryOrigin> origin = new DictionaryOrigin(prepath, nullptr); + // Create a cache entry for this if it doesn't exist. Note + // that the entry we're adding will need to be saved later once + // we have the entry + + // Open (and parse metadata) or create + RefPtr<DictionaryOriginReader> reader = new DictionaryOriginReader(); + reader->Start( + origin, prepath, aURI, this, + [origin, aDictEntry]( + DictionaryCacheEntry* + aDict) { // XXX avoid so many lambdas which cause allocations + // Write the dirty entry we couldn't write before once + // we get the hash + aDictEntry->WriteOnHash(origin); + return NS_OK; + }); + return origin; + }); + + newEntry = origin->AddEntry(aDictEntry, aNewEntry); return NS_OK; }); - return NS_OK; + return newEntry.forget(); } nsresult DictionaryCache::RemoveEntry(nsIURI* aURI, const nsACString& aKey) { @@ -481,18 +724,16 @@ nsresult DictionaryCache::RemoveEntry(nsIURI* aURI, const nsACString& aKey) { return NS_ERROR_FAILURE; } if (auto origin = mDictionaryCache.Lookup(prepath)) { - for (const auto& dict : *(origin->get())) { - if (dict->GetURI().Equals(aKey)) { - dict->remove(); - return NS_OK; - } - } - return NS_ERROR_FAILURE; + return origin.Data()->RemoveEntry(aKey); } return NS_ERROR_FAILURE; } -// return an entry +// Return an entry via a callback (async). +// If we don't have the origin in-memory, ask the cache for the origin, and +// when we get it, parse the metadata to build a DictionaryOrigin. +// Once we have a DictionaryOrigin (in-memory or parsed), scan it for matches. +// If it's not in the cache, return nullptr via callback. void DictionaryCache::GetDictionaryFor( nsIURI* aURI, const std::function<nsresult(DictionaryCacheEntry*)>& aCallback) { @@ -506,21 +747,13 @@ void DictionaryCache::GetDictionaryFor( } if (auto origin = mDictionaryCache.Lookup(prepath)) { // Find the longest match - uint32_t longest = 0; nsCString path; RefPtr<DictionaryCacheEntry> result; aURI->GetPathQueryRef(path); + DICTIONARY_LOG(("GetDictionaryFor(%s %s)", prepath.get(), path.get())); - for (const auto& dict : *(origin->get())) { - if (dict->Match(path, longest)) { - result = dict; - } - } - if (result) { - result->removeFrom(*(origin->get())); - origin->get()->insertFront(result); - } + result = origin.Data()->Match(path); (aCallback)(result); return; } @@ -529,78 +762,188 @@ void DictionaryCache::GetDictionaryFor( // in the memory cache. // Handle unknown origins by checking the disk cache - if (!sCacheStorage) { // in case we have no disk storage + if (!sCacheStorage) { + (aCallback)(nullptr); // in case we have no disk storage return; } // To keep track of the callback, we need a new object to get the // OnCacheEntryAvailable can resolve the callback. It will keep a reference // to itself until it call aCallback after the cache read has resolved - nsCString key("dict:"_ns + prepath); + DICTIONARY_LOG( + ("Reading %s for dictionary entries", PromiseFlatCString(prepath).get())); RefPtr<DictionaryOriginReader> reader = new DictionaryOriginReader(); - reader->Init(key, aCallback); + // After Start(), if we drop this ref reader will kill itself on + // completion; it holds a self-ref + reader->Start(nullptr, prepath, aURI, this, aCallback); } +//----------------------------------------------------------------------------- +// DictionaryOrigin +//----------------------------------------------------------------------------- +NS_IMPL_ISUPPORTS(DictionaryOrigin, nsICacheEntryMetaDataVisitor) -static void MakeMetadataEntry() { - // XXX +void DictionaryOrigin::Write(DictionaryCacheEntry* aDictEntry) { + DICTIONARY_LOG(("DictionaryOrigin::Write %s %p", mOrigin.get(), aDictEntry)); + aDictEntry->Write(mEntry); } -bool DictionaryCache::ParseMetaDataEntry(const char* key, const char* value, - nsCString& uri, uint32_t& hitCount, - uint32_t& lastHit, uint32_t& flags) { - MOZ_ASSERT(NS_IsMainThread()); - - DICTIONARY_LOG(("Dictionary::ParseMetaDataEntry key=%s value=%s", - key ? key : "", value)); - - const char* comma = strchr(value, ','); - if (!comma) { - DICTIONARY_LOG((" could not find first comma")); - return false; +already_AddRefed<DictionaryCacheEntry> DictionaryOrigin::AddEntry( + DictionaryCacheEntry* aDictEntry, bool aNewEntry) { + // Remove any entry for the same item + for (size_t i = 0; i < mEntries.Length(); i++) { + if (mEntries[i]->GetURI().Equals(aDictEntry->GetURI())) { + DictionaryCacheEntry* oldEntry = mEntries[i]; + if (aNewEntry) { + // We're overwriting an existing entry, perhaps with a new hash. It + // might be different, of course. + // Until we receive and save the new data, we should use the old data. + + // We need to pause this channel, regardless of how it's encoded, + // until the entry we're replacing has either no users, or has data + // read in from the cache. Then we can un-Suspend and start + // replacing the data in the cache itself. If there are no current + // users, and we start replacing the data, we need to remove the + // old entry so no one tries to use the old data/hash for a new + // request. + + // Note that when we start replacing data in the cache we need to + // also remove it from the origin's entry in the cache, in case we + // exit or crash before we finish replacing the entry and updating + // the origin's entry with the new hash. + + // Once we've replaced the entry (which will be after we have + // hash), new requests will use the new data/hash. I.e. we'll + // still allow new requests to use the old cache data/hash until + // the swap occurs. Once the swap happens, the channels using the + // old data/hash will still have an mDictDecoding reference to the + // DictionaryCacheEntry for the old data/hash. + + // XXX possible edge case: if a second request to replace the + // entry appears. Is this possible, or would the second request + // for the same URI get subsumed into the older one still in + // process? I'm guessing it doesn't, so we may need to deal with this + + DICTIONARY_LOG(( + "Replacing dictionary %p for %s: new will be %p", mEntries[i].get(), + PromiseFlatCString(oldEntry->GetURI()).get(), oldEntry)); + // May be overkill to check HasHash here + if (mEntries[i]->IsReading() && !aDictEntry->HasHash()) { + DICTIONARY_LOG(("Old entry is reading data")); + // If the old entry doesn't already have the data from the + // dictionary, we'll need to Suspend this channel, and do a + // replace later. Remember this new entry so when the current + // entry has it's data in memory we can un-Suspend the new + // channel/entry. When the new entry finishes saving, it will + // use the mReplacement link to come back and insert itself + // into mEntries and drop the old entry. Use an origin link + // for that since the old entry could in theory get purged and + // removed from the origin before we finish. + mEntries[i]->SetReplacement(aDictEntry, this); + // SetReplacement will also set aDictEntry->mShouldSuspend + return do_AddRef(aDictEntry); + } else { + DICTIONARY_LOG(("Removing old entry, no users or already read data")); + // We can just replace, there are no users active for the old data. + // This stops new requests from trying to use the old data we're in + // the process of replacing Remove the entry from the Origin and + // Write(). + mEntries[i]->RemoveEntry(mEntry); + mEntries.RemoveElementAt(i); + } + } else { + // We're updating an existing entry (on a 304 Not Modified). Assume + // the values may have changed (though likely they haven't). Check Spec + // XXX + DICTIONARY_LOG( + ("Updating dictionary for %s (%p)", mOrigin.get(), oldEntry)); + oldEntry->CopyFrom(aDictEntry); + // write the entry back if something changed + // XXX Check if something changed + oldEntry->Write(mEntry); + + // We don't need to reference the entry + return nullptr; + } + break; + } } - uint32_t version = static_cast<uint32_t>(atoi(value)); - DICTIONARY_LOG((" version -> %u", version)); - - if (version != METADATA_DICTIONARY_VERSION) { - DICTIONARY_LOG((" metadata version mismatch %u != %u", version, - METADATA_DICTIONARY_VERSION)); - return false; + DICTIONARY_LOG(("New dictionary %sfor %s: %p", + aDictEntry->HasHash() ? "" : "(pending) ", mOrigin.get(), + aDictEntry)); + if (aDictEntry->HasHash()) { + mEntries.AppendElement(aDictEntry); + } else { + // Still need to receive the data. When we have the hash, move to + // mEntries (and Write) using entry->mOrigin + mPendingEntries.AppendElement(aDictEntry); + aDictEntry->SetReplacement(nullptr, this); } - value = comma + 1; - comma = strchr(value, ','); - if (!comma) { - DICTIONARY_LOG((" could not find second comma")); - return false; - } + // DictionaryCache/caller is responsible for ensure this gets written if + // needed + return do_AddRef(aDictEntry); +} - hitCount = static_cast<uint32_t>(atoi(value)); - DICTIONARY_LOG((" hitCount -> %u", hitCount)); +nsresult DictionaryOrigin::RemoveEntry(const nsACString& aKey) { + for (const auto& dict : mEntries) { + if (dict->GetURI().Equals(aKey)) { + mEntries.RemoveElement(dict); + dict->RemoveEntry(mEntry); + return NS_OK; + } + } + return NS_ERROR_FAILURE; +} - value = comma + 1; - comma = strchr(value, ','); - if (!comma) { - DICTIONARY_LOG((" could not find third comma")); - return false; +void DictionaryOrigin::FinishAddEntry(DictionaryCacheEntry* aEntry) { + // if aDictEntry is in mPendingEntries, move to mEntries + if (mPendingEntries.RemoveElement(aEntry)) { + // We need to give priority to elements fetched most recently if they + // have an equivalent match length (and dest) + mEntries.InsertElementAt(0, aEntry); } +} - lastHit = static_cast<uint32_t>(atoi(value)); - DICTIONARY_LOG((" lastHit -> %u", lastHit)); +void DictionaryOrigin::RemoveEntry(DictionaryCacheEntry* aEntry) { + mEntries.RemoveElement(aEntry); +} - value = comma + 1; - flags = static_cast<uint32_t>(atoi(value)); - DICTIONARY_LOG((" flags -> %u", flags)); +// caller will throw this into a RefPtr +DictionaryCacheEntry* DictionaryOrigin::Match(const nsACString& aPath) { + uint32_t longest = 0; + DictionaryCacheEntry* result = nullptr; - if (key) { - const char* uriStart = key + (sizeof(META_DICTIONARY_PREFIX) - 1); - uri.AssignASCII(uriStart); - DICTIONARY_LOG((" uri -> %s", uriStart)); - } else { - uri.Truncate(); + for (const auto& dict : mEntries) { + if (dict->Match(aPath, longest)) { + result = dict; + } + } + // XXX if we want to LRU the origins so we can push them out of memory based + // on LRU, do something like this: + /* + if (result) { + removeFrom(dictionarycase->mDictionaryCacheLRU); + dictionarycase->mDictionaryCacheLRU.insertFront(this); } + */ + return result; +} - return true; +//----------------------------------------------------------------------------- +// DictionaryOrigin::nsICacheEntryMetaDataVisitor +//----------------------------------------------------------------------------- +nsresult DictionaryOrigin::OnMetaDataElement(const char* asciiKey, + const char* asciiValue) { + DICTIONARY_LOG(("DictionaryOrigin::OnMetaDataElement %s %s", + asciiKey ? asciiKey : "", asciiValue)); + + RefPtr<DictionaryCacheEntry> entry = new DictionaryCacheEntry(asciiKey); + if (entry->ParseMetadata(asciiValue)) { + // XXX Check for duplicates? + mEntries.AppendElement(entry); + } + return NS_OK; } // Overall structure: diff --git a/netwerk/cache2/Dictionary.h b/netwerk/cache2/Dictionary.h @@ -25,7 +25,6 @@ class nsICacheStorage; class nsIIOService; class nsILoadContextInfo; -class nsITimer; // Version of metadata entries we expect static const uint32_t METADATA_DICTIONARY_VERSION = 1; @@ -34,6 +33,8 @@ static const uint32_t METADATA_DICTIONARY_VERSION = 1; namespace mozilla { namespace net { +class DictionaryOrigin; + // Outstanding requests that offer this dictionary will hold a reference to it. // If it's replaced (or removed) during the request, we would a) read the data // into memory* b) unlink this from the origin in the memory cache. @@ -43,15 +44,10 @@ namespace net { // // When creating an entry from incoming data, we'll create it with no hash // initially until the full data has arrived, then update the Hash. -class DictionaryCacheEntry final - : public LinkedListElement<RefPtr<DictionaryCacheEntry>>, - public nsICacheEntryOpenCallback, - public nsIStreamListener { +class DictionaryCacheEntry final : public nsICacheEntryOpenCallback, + public nsIStreamListener { private: - ~DictionaryCacheEntry() { - MOZ_ASSERT(mUsers == 0); - MOZ_ASSERT(!isInList()); - } + ~DictionaryCacheEntry() { MOZ_ASSERT(mUsers == 0); } public: NS_DECL_THREADSAFE_ISUPPORTS @@ -59,6 +55,7 @@ class DictionaryCacheEntry final NS_DECL_NSIREQUESTOBSERVER NS_DECL_NSISTREAMLISTENER + DictionaryCacheEntry(const char* aKey); DictionaryCacheEntry(const nsACString& aKey, const nsACString& aPattern, const nsACString& aId, const Maybe<nsCString>& aHash = Nothing()); @@ -73,16 +70,43 @@ class DictionaryCacheEntry final const nsACString& GetHash() const { return mHash; } + bool HasHash() { + // Hard to statically check since we're called from lambdas in + // GetDictionaryFor + return !mHash.IsEmpty(); + } + void SetHash(const nsACString& aHash) { MOZ_ASSERT(NS_IsMainThread()); mHash = aHash; } + void WriteOnHash(DictionaryOrigin* aOrigin); + const nsCString& GetId() const { return mId; } // keep track of requests that may need the data void InUse(); void UseCompleted(); + bool IsReading() const { return mUsers > 0 && !mWaitingPrefetch.IsEmpty(); } + + void SetReplacement(DictionaryCacheEntry* aEntry, DictionaryOrigin* aOrigin) { + mReplacement = aEntry; + mOrigin = aOrigin; + if (mReplacement) { + mReplacement->mShouldSuspend = true; + mReplacement->mBlocked = true; + } + } + + bool ShouldSuspendUntilCacheRead() const { return mShouldSuspend; } + + // aFunc is called when we have finished reading a dictionary from the + // cache, or we have no users waiting for cache data (cancelled, etc) + void CallbackOnCacheRead(const std::function<void()>& aFunc) { + // the reasons to call back are identical to Prefetch() + mWaitingPrefetch.AppendElement(aFunc); + } const nsACString& GetURI() const { return mURI; } @@ -96,10 +120,9 @@ class DictionaryCacheEntry final const Vector<uint8_t>& GetDictionary() const { return mDictionaryData; } + // Accumulate a hash while saving a file being received to the cache void AccumulateHash(const char* aBuf, int32_t aCount); - void AccumulateFile(const char* aBuf, int32_t aCount); - - void FinishFile(); + void FinishHash(); // return a pointer to the data and length uint8_t* DictionaryData(size_t* aLength) const { @@ -118,10 +141,32 @@ class DictionaryCacheEntry final const char* aFromSegment, uint32_t aToOffset, uint32_t aCount, uint32_t* aWriteCount); + void MakeMetadataEntry(nsCString& aNewValue); + + nsresult Write(nsICacheEntry* aEntry); + + nsresult RemoveEntry(nsICacheEntry* aCacheEntry); + + // Parse metadata from DictionaryOrigin + bool ParseMetadata(const char* aSrc); + + void CopyFrom(DictionaryCacheEntry* aOther) { + mURI = aOther->mURI; + mPattern = aOther->mPattern; + mId = aOther->mId; + // XXX match-dest + // XXX type + } + + void UnblockAddEntry(DictionaryOrigin* aOrigin); + private: nsCString mURI; // URI (without ref) for the dictionary nsCString mPattern; nsCString mId; // max length 1024 + // XXX add list of match-dest values + // XXX add type + // dcb and dcz use type 'raw'. We're allowed to ignore types we don't // understand, so we can fail to record a dictionary with type != 'raw' // nsCString mType; @@ -138,6 +183,24 @@ class DictionaryCacheEntry final // call these when prefetch is complete nsTArray<std::function<void()>> mWaitingPrefetch; + + // If we need to Write() an entry before we know the hash, remember the origin + // here (creates a temporary cycle). Clear on StopRequest + RefPtr<DictionaryOrigin> mOrigin; + // Don't store origin for write if we've already received OnStopRequest + bool mStopReceived{false}; + + // If set, a new entry wants to replace us, and we have active decoding users. + // When we finish reading data into this entry for decoding, do 2 things: + // Remove our entry from origin->mEntries (so no future requests find this, + // and un-Suspend the new channel so it can start saving data into the cache. + RefPtr<DictionaryCacheEntry> mReplacement; + + // We should suspend until the ond entry has been read + bool mShouldSuspend{false}; + + // We're blocked from taking over for the old entry for now + bool mBlocked{false}; }; // XXX Do we want to pre-read dictionaries into RAM at startup (lazily)? @@ -150,21 +213,54 @@ class DictionaryCacheEntry final // XXX Clear all dictionaries when cookies are cleared for a site -using DictCacheList = AutoCleanLinkedList<RefPtr<DictionaryCacheEntry>>; +// using DictCacheList = AutoCleanLinkedList<RefPtr<DictionaryCacheEntry>>; +using DictCacheList = nsTArray<RefPtr<DictionaryCacheEntry>>; + +// XXX if we want to have a parallel LRU list for pushing origins out of memory, +// add this: public LinkedListElement<RefPtr<DictionaryOrigin>>, +class DictionaryOrigin : public nsICacheEntryMetaDataVisitor { + friend class DictionaryCache; + + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICACHEENTRYMETADATAVISITOR + + DictionaryOrigin(const nsACString& aOrigin, nsICacheEntry* aEntry) + : mOrigin(aOrigin), mEntry(aEntry) {} + + void SetCacheEntry(nsICacheEntry* aEntry) { mEntry = aEntry; } + void Write(DictionaryCacheEntry* aDictEntry); + already_AddRefed<DictionaryCacheEntry> AddEntry( + DictionaryCacheEntry* aDictEntry, bool aNewEntry); + nsresult RemoveEntry(const nsACString& aKey); + void RemoveEntry(DictionaryCacheEntry* aEntry); + DictionaryCacheEntry* Match(const nsACString& path); + void FinishAddEntry(DictionaryCacheEntry* aEntry); + + private: + virtual ~DictionaryOrigin() {} + + nsCString mOrigin; + nsCOMPtr<nsICacheEntry> mEntry; + DictCacheList mEntries; + // Dictionaries currently being received. Once these get a Hash, move to + // mEntries + DictCacheList mPendingEntries; +}; class DictionaryOriginReader; // singleton class class DictionaryCache final { private: - DictionaryCache() { Init(); }; - ~DictionaryCache() {}; + DictionaryCache() { Init(); } + ~DictionaryCache() {} friend class DictionaryOriginReader; friend class DictionaryCacheEntry; public: - NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DictionaryCache) + NS_INLINE_DECL_REFCOUNTING(DictionaryCache) static already_AddRefed<DictionaryCache> GetInstance(); @@ -172,10 +268,11 @@ class DictionaryCache final { nsresult AddEntry(nsIURI* aURI, const nsACString& aKey, const nsACString& aPattern, const nsACString& aId, - const Maybe<nsCString>& aHash, + const Maybe<nsCString>& aHash, bool aNewEntry, DictionaryCacheEntry** aDictEntry); - nsresult AddEntry(nsIURI* aURI, DictionaryCacheEntry* aDictEntry); + already_AddRefed<DictionaryCacheEntry> AddEntry( + nsIURI* aURI, bool aNewEntry, DictionaryCacheEntry* aDictEntry); nsresult RemoveEntry(nsIURI* aURI, const nsACString& aKey); @@ -190,10 +287,6 @@ class DictionaryCache final { } private: - bool ParseMetaDataEntry(const char* key, const char* value, nsCString& uri, - uint32_t& hitCount, uint32_t& lastHit, - uint32_t& flags); - static nsCOMPtr<nsICacheStorage> sCacheStorage; // In-memory cache of dictionary entries. HashMap, keyed by origin, of @@ -202,7 +295,7 @@ class DictionaryCache final { // if there are dictionaries for an origin. // Static assertions fire if we try to have a LinkedList directly in an // nsTHashMap - nsTHashMap<nsCStringHashKey, UniquePtr<DictCacheList>> mDictionaryCache; + nsTHashMap<nsCStringHashKey, RefPtr<DictionaryOrigin>> mDictionaryCache; }; } // namespace net diff --git a/netwerk/protocol/http/HttpBaseChannel.h b/netwerk/protocol/http/HttpBaseChannel.h @@ -874,10 +874,6 @@ class HttpBaseChannel : public nsHashPropertyBag, nsIRequest::TRRMode mEffectiveTRRMode = nsIRequest::TRR_DEFAULT_MODE; TRRSkippedReason mTRRSkipReason = TRRSkippedReason::TRR_UNSET; - // Dictionary entry - retain while we're saving the data, and while we're - // fetching data possibly encoded with the entry - RefPtr<DictionaryCacheEntry> mDict; - public: void SetEarlyHints( nsTArray<mozilla::net::EarlyHintConnectArgs>&& aEarlyHints); diff --git a/netwerk/protocol/http/HttpChannelChild.h b/netwerk/protocol/http/HttpChannelChild.h @@ -84,11 +84,13 @@ class HttpChannelChild final : public PHttpChannelChild, // nsIChannel NS_IMETHOD GetSecurityInfo(nsITransportSecurityInfo** aSecurityInfo) override; NS_IMETHOD AsyncOpen(nsIStreamListener* aListener) override; - NS_IMETHOD GetDictionary(DictionaryCacheEntry** aDictionary) override { + NS_IMETHOD GetDecompressDictionary( + DictionaryCacheEntry** aDictionary) override { *aDictionary = nullptr; return NS_OK; } - NS_IMETHOD SetDictionary(DictionaryCacheEntry* aDictionary) override { + NS_IMETHOD SetDecompressDictionary( + DictionaryCacheEntry* aDictionary) override { return NS_OK; } diff --git a/netwerk/protocol/http/InterceptedHttpChannel.h b/netwerk/protocol/http/InterceptedHttpChannel.h @@ -320,11 +320,13 @@ class InterceptedHttpChannel final void DoAsyncAbort(nsresult aStatus) override; - NS_IMETHOD GetDictionary(DictionaryCacheEntry** aDictionary) override { + NS_IMETHOD GetDecompressDictionary( + DictionaryCacheEntry** aDictionary) override { *aDictionary = nullptr; return NS_OK; } - NS_IMETHOD SetDictionary(DictionaryCacheEntry* aDictionary) override { + NS_IMETHOD SetDecompressDictionary( + DictionaryCacheEntry* aDictionary) override { return NS_OK; } }; diff --git a/netwerk/protocol/http/NullHttpChannel.cpp b/netwerk/protocol/http/NullHttpChannel.cpp @@ -847,14 +847,14 @@ NullHttpChannel::GetRenderBlocking(bool* aRenderBlocking) { } NS_IMETHODIMP -NullHttpChannel::GetDictionary( +NullHttpChannel::GetDecompressDictionary( mozilla::net::DictionaryCacheEntry** aDictionary) { *aDictionary = nullptr; return NS_OK; } NS_IMETHODIMP -NullHttpChannel::SetDictionary( +NullHttpChannel::SetDecompressDictionary( mozilla::net::DictionaryCacheEntry* aDictionary) { return NS_OK; } diff --git a/netwerk/protocol/http/ObliviousHttpChannel.cpp b/netwerk/protocol/http/ObliviousHttpChannel.cpp @@ -880,13 +880,13 @@ NS_IMETHODIMP ObliviousHttpChannel::GetDocumentCharacterSet( return NS_ERROR_NOT_IMPLEMENTED; } -NS_IMETHODIMP ObliviousHttpChannel::GetDictionary( +NS_IMETHODIMP ObliviousHttpChannel::GetDecompressDictionary( DictionaryCacheEntry** aDictionary) { *aDictionary = nullptr; return NS_OK; } -NS_IMETHODIMP ObliviousHttpChannel::SetDictionary( +NS_IMETHODIMP ObliviousHttpChannel::SetDecompressDictionary( DictionaryCacheEntry* aDictionary) { return NS_OK; } diff --git a/netwerk/protocol/http/PHttpChannelParams.h b/netwerk/protocol/http/PHttpChannelParams.h @@ -115,6 +115,10 @@ struct ParamTraits<mozilla::net::nsHttpHeaderArray::nsEntry> { break; case mozilla::net::nsHttpHeaderArray::eVarietyResponse: WriteParam(aWriter, (uint8_t)6); + break; + case mozilla::net::nsHttpHeaderArray::eVarietyResponseOverride: + WriteParam(aWriter, (uint8_t)7); + break; } } @@ -158,6 +162,10 @@ struct ParamTraits<mozilla::net::nsHttpHeaderArray::nsEntry> { case 6: aResult->variety = mozilla::net::nsHttpHeaderArray::eVarietyResponse; break; + case 7: + aResult->variety = + mozilla::net::nsHttpHeaderArray::eVarietyResponseOverride; + break; default: return false; } diff --git a/netwerk/protocol/http/TRRServiceChannel.h b/netwerk/protocol/http/TRRServiceChannel.h @@ -76,11 +76,13 @@ class TRRServiceChannel : public HttpBaseChannel, NS_IMETHOD SetNotificationCallbacks( nsIInterfaceRequestor* aCallbacks) override; - NS_IMETHOD GetDictionary(DictionaryCacheEntry** aDictionary) override { + NS_IMETHOD GetDecompressDictionary( + DictionaryCacheEntry** aDictionary) override { *aDictionary = nullptr; return NS_OK; } - NS_IMETHOD SetDictionary(DictionaryCacheEntry* aDictionary) override { + NS_IMETHOD SetDecompressDictionary( + DictionaryCacheEntry* aDictionary) override { return NS_OK; } // nsISupportsPriority diff --git a/netwerk/protocol/http/nsHttpChannel.cpp b/netwerk/protocol/http/nsHttpChannel.cpp @@ -505,8 +505,8 @@ nsHttpChannel::~nsHttpChannel() { gHttpHandler->RemoveHttpChannel(mChannelId); } - if (mDict && mUsingDictionary) { - mDict->UseCompleted(); + if (mDictDecompress && mUsingDictionary) { + mDictDecompress->UseCompleted(); } } @@ -554,16 +554,19 @@ nsresult nsHttpChannel::Init(nsIURI* uri, uint32_t caps, nsProxyInfo* proxyInfo, rv = gHttpHandler->AddAcceptAndDictionaryHeaders( uri, &mRequestHead, IsHTTPS(), [self = RefPtr(this)](DictionaryCacheEntry* aDict) { - self->mDict = aDict; - if (self->mDict) { - // mDict is set if we added Available-Dictionary - self->mDict->InUse(); + self->mDictDecompress = aDict; + if (self->mDictDecompress) { + LOG_DICTIONARIES( + ("Added dictionary header for %p, DirectoryCacheEntry %p", + self.get(), aDict)); + // mDictDecompress is set if we added Available-Dictionary + self->mDictDecompress->InUse(); self->mUsingDictionary = true; - self->mShouldSuspendForDictionary = - self->mDict->Prefetch(GetLoadContextInfo(self), [self]() { + self->mShouldSuspendForDictionary = self->mDictDecompress->Prefetch( + GetLoadContextInfo(self), [self]() { // this is called when the prefetch is complete to // un-Suspend the channel - MOZ_ASSERT(self->mDict->DictionaryReady()); + MOZ_ASSERT(self->mDictDecompress->DictionaryReady()); if (self->mSuspendedForDictionary) { LOG( ("nsHttpChannel::Init [this=%p] Resuming channel " @@ -3524,6 +3527,16 @@ nsresult nsHttpChannel::ContinueProcessNormal(nsresult rv) { if (mCacheEntry) { rv = InitCacheEntry(); if (NS_FAILED(rv)) CloseCacheEntry(true); + } + + // We need to do this before CallonStartRequest, since this can modify + // the Content-Encoding to remove dcb/dcz (and perhaps others), and + // CallOnStartRequest() sends this to the content process. + + // Install cache listener if we still have a cache entry open + if (mCacheEntry && !LoadCacheEntryIsReadOnly()) { + rv = InstallCacheListener(); + if (NS_FAILED(rv)) return rv; } else { // We may still need to decompress in the parent if it's dcb or dcz nsAutoCString contentEncoding; @@ -3552,6 +3565,7 @@ nsresult nsHttpChannel::ContinueProcessNormal(nsresult rv) { } } } + // Check that the server sent us what we were asking for if (LoadResuming()) { // Create an entity id from the response @@ -3580,8 +3594,8 @@ nsresult nsHttpChannel::ContinueProcessNormal(nsresult rv) { // If we don't have the entire dictionary yet, Suspend() the channel // until the dictionary is in-memory. - if (mDict && mUsingDictionary && mShouldSuspendForDictionary && - !mDict->DictionaryReady()) { + if (mDictDecompress && mUsingDictionary && mShouldSuspendForDictionary && + !mDictDecompress->DictionaryReady()) { LOG( ("nsHttpChannel::ContinueProcessNormal [this=%p] Suspending the " "transaction, waiting for dictionary", @@ -3590,16 +3604,6 @@ nsresult nsHttpChannel::ContinueProcessNormal(nsresult rv) { mSuspendedForDictionary = true; } - // We need to do this before CallonStartRequest, since this can modify - // the Content-Encoding to remove dcb/dcz (and perhaps others), and - // CallOnStartRequest() sends this to the content process. - - // Install cache listener if we still have a cache entry open - if (mCacheEntry && !LoadCacheEntryIsReadOnly()) { - rv = InstallCacheListener(); - if (NS_FAILED(rv)) return rv; - } - rv = CallOnStartRequest(); if (NS_FAILED(rv)) return rv; @@ -4468,7 +4472,7 @@ nsresult nsHttpChannel::ProcessNotModified( rv = UpdateExpirationTime(); if (NS_FAILED(rv)) return rv; - rv = AddCacheEntryHeaders(mCacheEntry); + rv = AddCacheEntryHeaders(mCacheEntry, false); if (NS_FAILED(rv)) return rv; // notify observers interested in looking at a reponse that has been @@ -5836,9 +5840,8 @@ nsresult nsHttpChannel::InitCacheEntry() { // mark this weakly framed until a response body is seen mCacheEntry->SetMetaDataElement("strongly-framed", "0"); - rv = AddCacheEntryHeaders(mCacheEntry); - if (NS_FAILED(rv)) return rv; - + // We'll store the cache headers in InstallCacheListener, since it may modify + // them StoreInitedCacheEntry(true); // Don't perform the check when writing (doesn't make sense) @@ -5872,7 +5875,8 @@ void nsHttpChannel::UpdateInhibitPersistentCachingFlag() { nsresult DoAddCacheEntryHeaders(nsHttpChannel* self, nsICacheEntry* entry, nsHttpRequestHead* requestHead, nsHttpResponseHead* responseHead, - nsITransportSecurityInfo* securityInfo) { + nsITransportSecurityInfo* securityInfo, + bool aModified) { nsresult rv; LOG(("nsHttpChannel::AddCacheEntryHeaders [this=%p] begin", self)); @@ -5881,6 +5885,11 @@ nsresult DoAddCacheEntryHeaders(nsHttpChannel* self, nsICacheEntry* entry, entry->SetSecurityInfo(securityInfo); } + // Note: if aModified == false, then we're processing a 304 Not Modified, + // and we *shouldn't* have any change to the Dictionary (and won't be + // replacing the DictionaryEntry, though if the Match/Match-dest/Id/Type + // changed, we may need to rewrite it. XXX? + // Store the HTTP request method with the cache entry so we can distinguish // for example GET and HEAD responses. nsAutoCString method; @@ -5966,7 +5975,10 @@ nsresult DoAddCacheEntryHeaders(nsHttpChannel* self, nsICacheEntry* entry, // If this is being marked as a dictionary, add it to the list if (StaticPrefs::network_http_dictionaries_enable() && self->IsHTTPS()) { - self->ParseDictionary(entry, responseHead); + if (!self->ParseDictionary(entry, responseHead, aModified)) { + LOG_DICTIONARIES( + ("Failed to parse use-as-dictionary from %s", head.get())); + } } // Indicate we have successfully finished setting metadata on the cache entry. @@ -5976,7 +5988,8 @@ nsresult DoAddCacheEntryHeaders(nsHttpChannel* self, nsICacheEntry* entry, } bool nsHttpChannel::ParseDictionary(nsICacheEntry* aEntry, - nsHttpResponseHead* aResponseHead) { + nsHttpResponseHead* aResponseHead, + bool aModified) { nsAutoCString val; if (NS_SUCCEEDED(aResponseHead->GetHeader(nsHttp::Use_As_Dictionary, val))) { nsAutoCStringN<128> matchVal; @@ -5994,6 +6007,7 @@ bool nsHttpChannel::ParseDictionary(nsICacheEntry* aEntry, if (NS_FAILED(rv = aEntry->GetKey(key))) { return false; } + nsCString hash; // Available now for use RefPtr<DictionaryCache> dicts(DictionaryCache::GetInstance()); @@ -6002,16 +6016,30 @@ bool nsHttpChannel::ParseDictionary(nsICacheEntry* aEntry, "type=%s", mURI->GetSpecOrDefault().get(), key.get(), matchVal.get(), matchIdVal.get(), typeVal.get())); - dicts->AddEntry(mURI, key, matchVal, matchIdVal, Some(hash), - getter_AddRefs(mDict)); + dicts->AddEntry(mURI, key, matchVal, matchIdVal, Some(hash), aModified, + getter_AddRefs(mDictSaving)); + // If this was 304 Not Modified, then we don't need the dictionary data + // (though we may update the dictionary entry if the match/id/etc changed). + // If this is 304, mDictSaving will be cleared by AddEntry. + if (mDictSaving) { + if (mDictSaving->ShouldSuspendUntilCacheRead()) { + LOG_DICTIONARIES(("Suspending %p to wait for cache read", this)); + mTransactionPump->Suspend(); + mDictSaving->CallbackOnCacheRead([self = RefPtr(this)]() { + LOG_DICTIONARIES(("Resuming %p after cache read", self.get())); + self->Resume(); + }); + } + } return true; } - return false; + return true; // succeeded, no use-as-dictionary } -nsresult nsHttpChannel::AddCacheEntryHeaders(nsICacheEntry* entry) { +nsresult nsHttpChannel::AddCacheEntryHeaders(nsICacheEntry* entry, + bool aModified) { return DoAddCacheEntryHeaders(this, entry, &mRequestHead, mResponseHead.get(), - mSecurityInfo); + mSecurityInfo, aModified); } inline void GetAuthType(const char* challenge, nsCString& authType) { @@ -6077,10 +6105,19 @@ nsresult nsHttpChannel::InstallCacheListener(int64_t offset) { // and improve hitrate. Downside is CPU use, complexity and perhaps delay, // maybe. nsAutoCString dictionary; - Unused << mResponseHead->GetHeader(nsHttp::Use_As_Dictionary, dictionary); - if (!dictionary.IsEmpty()) { + if (StaticPrefs::network_http_dictionaries_enable() && IsHTTPS()) { + Unused << mResponseHead->GetHeader(nsHttp::Use_As_Dictionary, dictionary); + if (!dictionary.IsEmpty()) { + // If this is being marked as a dictionary, add it to the list + if (!ParseDictionary(mCacheEntry, mResponseHead.get(), true)) { + LOG_DICTIONARIES(("Failed to parse use-as-dictionary")); + } else { + MOZ_ASSERT(mDictSaving); + } + } + // We need to record the hash as we save it - mCacheEntry->SetDictionary(mDict); + mCacheEntry->SetDictionary(mDictSaving); } LOG(("Trading cache input stream for output stream [channel=%p]", this)); @@ -6094,6 +6131,10 @@ nsresult nsHttpChannel::InstallCacheListener(int64_t offset) { predictedSize -= offset; } + nsCOMPtr<nsIStreamListenerTee> tee = + do_CreateInstance(kStreamListenerTeeCID, &rv); + if (NS_FAILED(rv)) return rv; + nsCOMPtr<nsIOutputStream> out; rv = mCacheEntry->OpenOutputStream(offset, predictedSize, getter_AddRefs(out)); @@ -6129,10 +6170,6 @@ nsresult nsHttpChannel::InstallCacheListener(int64_t offset) { if (NS_FAILED(rv)) return rv; #endif - nsCOMPtr<nsIStreamListenerTee> tee = - do_CreateInstance(kStreamListenerTeeCID, &rv); - if (NS_FAILED(rv)) return rv; - rv = tee->Init(mListener, out, nullptr); LOG(("nsHttpChannel::InstallCacheListener sync tee %p rv=%" PRIx32, tee.get(), static_cast<uint32_t>(rv))); @@ -6153,7 +6190,8 @@ nsresult nsHttpChannel::InstallCacheListener(int64_t offset) { ("Content-Encoding for %p: %s", this, contentEncoding.get())); if (!dictionary.IsEmpty() || contentEncoding.Equals("dcb") || contentEncoding.Equals("dcz")) { - LOG_DICTIONARIES(("Removing Content-Encoding for %p", this)); + LOG_DICTIONARIES( + ("Removing Content-Encoding %s for %p", contentEncoding.get(), this)); nsCOMPtr<nsIStreamListener> listener; // otherwise we won't convert in the parent process SetApplyConversion(true); @@ -6172,6 +6210,14 @@ nsresult nsHttpChannel::InstallCacheListener(int64_t offset) { LOG_DICTIONARIES(("Didn't install decompressor before tee")); } + // Must add these after adding possible decompressors, since that may modify + // Content-Encoding + rv = AddCacheEntryHeaders(mCacheEntry, true); + if (NS_FAILED(rv)) { + mCacheEntry->AsyncDoom(nullptr); + return rv; + } + // XXX telemetry as to how often we get dcb/dcz return NS_OK; } @@ -7787,19 +7833,24 @@ nsHttpChannel::GetEncodedBodySize(uint64_t* aEncodedBodySize) { } NS_IMETHODIMP -nsHttpChannel::GetDictionary(DictionaryCacheEntry** aDictionary) { - *aDictionary = do_AddRef(mDict).take(); +nsHttpChannel::GetDecompressDictionary(DictionaryCacheEntry** aDictionary) { + *aDictionary = do_AddRef(mDictDecompress).take(); return NS_OK; } NS_IMETHODIMP -nsHttpChannel::SetDictionary(DictionaryCacheEntry* aDictionary) { - if (mDict && mUsingDictionary) { - mDict->UseCompleted(); +nsHttpChannel::SetDecompressDictionary(DictionaryCacheEntry* aDictionary) { + if (!aDictionary) { + if (mDictDecompress && mUsingDictionary) { + mDictDecompress->UseCompleted(); + } mUsingDictionary = false; + } else { + MOZ_ASSERT(!mDictDecompress); + aDictionary->InUse(); + mUsingDictionary = true; } - mDict = aDictionary; - // We're only setting InUse() if we are using it for Available-Dictionary + mDictDecompress = aDictionary; return NS_OK; } diff --git a/netwerk/protocol/http/nsHttpChannel.h b/netwerk/protocol/http/nsHttpChannel.h @@ -205,8 +205,10 @@ class nsHttpChannel final : public HttpBaseChannel, NS_IMETHOD SetResponseStatus(uint32_t aStatus, const nsACString& aStatusText) override; - NS_IMETHOD GetDictionary(DictionaryCacheEntry** aDictionary) override; - NS_IMETHOD SetDictionary(DictionaryCacheEntry* aDictionary) override; + NS_IMETHOD GetDecompressDictionary( + DictionaryCacheEntry** aDictionary) override; + NS_IMETHOD SetDecompressDictionary( + DictionaryCacheEntry* aDictionary) override; void SetWarningReporter(HttpChannelSecurityWarningReporter* aReporter); HttpChannelSecurityWarningReporter* GetWarningReporter(); @@ -415,9 +417,10 @@ class nsHttpChannel final : public HttpBaseChannel, void CloseCacheEntry(bool doomOnFailure); [[nodiscard]] nsresult InitCacheEntry(); void UpdateInhibitPersistentCachingFlag(); - bool ParseDictionary(nsICacheEntry* aEntry, - nsHttpResponseHead* aResponseHead); - [[nodiscard]] nsresult AddCacheEntryHeaders(nsICacheEntry* entry); + bool ParseDictionary(nsICacheEntry* aEntry, nsHttpResponseHead* aResponseHead, + bool aModified); + [[nodiscard]] nsresult AddCacheEntryHeaders(nsICacheEntry* entry, + bool aModified); [[nodiscard]] nsresult FinalizeCacheEntry(); [[nodiscard]] nsresult InstallCacheListener(int64_t offset = 0); void MaybeInvalidateCacheEntryForSubsequentGet(); @@ -560,6 +563,16 @@ class nsHttpChannel final : public HttpBaseChannel, // state of whether tracking protection is enabled or not. RefPtr<nsChannelClassifier> mChannelClassifier; + // Dictionary entry for the entry being used to decompress this stream + // (i.e. we added Dictionary-Available to the request). + RefPtr<DictionaryCacheEntry> mDictDecompress; + // This is for channels we're going to use a dictionaries in the future + // (i.e. ResponseHeaders has Use-As-Dictionary) + RefPtr<DictionaryCacheEntry> mDictSaving; + // Note that in the case of using a file to be a dictionary for future + // versions of itself, these may have the same URI (but likely different + // hashes). + // Proxy release all members above on main thread. void ReleaseMainThreadOnlyReferences(); diff --git a/netwerk/protocol/http/nsHttpHandler.cpp b/netwerk/protocol/http/nsHttpHandler.cpp @@ -694,6 +694,8 @@ nsresult nsHttpHandler::AddAcceptAndDictionaryHeaders( return rv; } if (!aDict->GetId().IsEmpty()) { + LOG_DICTIONARIES(("Setting Dictionary-Id: %s", + PromiseFlatCString(aDict->GetId()).get())); rv = aRequest->SetHeader( nsHttp::Dictionary_Id, aDict->GetId(), false, nsHttpHeaderArray::eVarietyRequestOverride); @@ -701,8 +703,6 @@ nsresult nsHttpHandler::AddAcceptAndDictionaryHeaders( (aCallback)(nullptr); return rv; } - LOG_DICTIONARIES( - ("Setting Dictionary-Id: %s", aDict->GetId().get())); } // Need to retain access to the dictionary until the request // completes. Note that this includes if the dictionary we offered diff --git a/netwerk/protocol/http/nsHttpHeaderArray.cpp b/netwerk/protocol/http/nsHttpHeaderArray.cpp @@ -43,14 +43,17 @@ nsresult nsHttpHeaderArray::SetHeader( MOZ_ASSERT( (variety == eVarietyResponse) || (variety == eVarietyRequestDefault) || (variety == eVarietyRequestOverride) || + (variety == eVarietyResponseOverride) || (variety == eVarietyRequestEnforceDefault), "Net original headers can only be set using SetHeader_internal()."); nsEntry* entry = nullptr; int32_t index = LookupEntry(header, &entry); - // If an empty value is received and we aren't merging headers discard it - if (value.IsEmpty() && header != nsHttp::X_Frame_Options) { + // If an empty value is received and we aren't merging headers discard it, + // unless we're overriding + if (value.IsEmpty() && header != nsHttp::X_Frame_Options && + variety != eVarietyResponseOverride) { if (!merge && entry) { if (entry->variety == eVarietyResponseNetOriginalAndResponse) { MOZ_ASSERT(variety == eVarietyResponse); @@ -79,7 +82,8 @@ nsresult nsHttpHeaderArray::SetHeader( if (!IsIgnoreMultipleHeader(header)) { // Replace the existing string with the new value if (entry->variety == eVarietyResponseNetOriginalAndResponse) { - MOZ_ASSERT(variety == eVarietyResponse); + MOZ_ASSERT(variety == eVarietyResponse || + variety == eVarietyResponseOverride); entry->variety = eVarietyResponseNetOriginal; return SetHeader_internal(header, headerName, value, variety); } diff --git a/netwerk/protocol/http/nsHttpHeaderArray.h b/netwerk/protocol/http/nsHttpHeaderArray.h @@ -46,7 +46,8 @@ class nsHttpHeaderArray { // Used only for response header. eVarietyResponseNetOriginalAndResponse, eVarietyResponseNetOriginal, - eVarietyResponse + eVarietyResponse, + eVarietyResponseOverride, }; // Used by internal setters: to set header from network use SetHeaderFromNet diff --git a/netwerk/protocol/http/nsHttpResponseHead.cpp b/netwerk/protocol/http/nsHttpResponseHead.cpp @@ -202,7 +202,8 @@ nsresult nsHttpResponseHead::SetHeaderOverride(const nsHttpAtom& atom, return NS_ERROR_FAILURE; } - return mHeaders.SetHeaderFromNet(atom, ""_ns, val, true); + return mHeaders.SetHeader(atom, ""_ns, val, false, + nsHttpHeaderArray::eVarietyResponseOverride); } nsresult nsHttpResponseHead::SetHeader_locked(const nsHttpAtom& atom, diff --git a/netwerk/protocol/http/nsIHttpChannel.idl b/netwerk/protocol/http/nsIHttpChannel.idl @@ -516,7 +516,5 @@ interface nsIHttpChannel : nsIIdentChannel /** * Dictionary for decompression, if any */ - [noscript] attribute DictionaryCacheEntry dictionary; - - + [noscript] attribute DictionaryCacheEntry decompressDictionary; }; diff --git a/netwerk/protocol/viewsource/nsViewSourceChannel.cpp b/netwerk/protocol/viewsource/nsViewSourceChannel.cpp @@ -1225,14 +1225,14 @@ nsViewSourceChannel::AsyncOnChannelRedirect( } NS_IMETHODIMP -nsViewSourceChannel::GetDictionary( +nsViewSourceChannel::GetDecompressDictionary( mozilla::net::DictionaryCacheEntry** aDictionary) { *aDictionary = nullptr; return NS_OK; } NS_IMETHODIMP -nsViewSourceChannel::SetDictionary( +nsViewSourceChannel::SetDecompressDictionary( mozilla::net::DictionaryCacheEntry* aDictionary) { return NS_OK; } diff --git a/netwerk/streamconv/converters/nsHTTPCompressConv.cpp b/netwerk/streamconv/converters/nsHTTPCompressConv.cpp @@ -62,7 +62,8 @@ class BrotliWrapper { return false; } // XXX Wait for dictionary to be read into RAM!! - if (NS_SUCCEEDED(httpchannel->GetDictionary(getter_AddRefs(mDictionary))) && + if (NS_SUCCEEDED(httpchannel->GetDecompressDictionary( + getter_AddRefs(mDictionary))) && mDictionary) { size_t length = mDictionary->GetDictionary().length(); DICTIONARY_LOG(("Brotli: dictionary %zu bytes", length)); @@ -111,8 +112,8 @@ class ZstdWrapper { nsCOMPtr<nsIHttpChannel> httpchannel(do_QueryInterface(aRequest)); if (httpchannel) { // XXX Wait for dictionary to be read into RAM!! - if (NS_FAILED( - httpchannel->GetDictionary(getter_AddRefs(mDictionary))) || + if (NS_FAILED(httpchannel->GetDecompressDictionary( + getter_AddRefs(mDictionary))) || !mDictionary) { return; } @@ -367,7 +368,7 @@ nsHTTPCompressConv::OnStopRequest(nsIRequest* request, nsresult aStatus) { RefPtr<DictionaryCacheEntry> dict; nsCOMPtr<nsIHttpChannel> httpchannel(do_QueryInterface(request)); if (httpchannel) { - httpchannel->SetDictionary(nullptr); + httpchannel->SetDecompressDictionary(nullptr); } // paranoia mBrotli = nullptr;