tor-browser

The Tor Browser
git clone https://git.dasho.dev/tor-browser.git
Log | Files | Refs | README | LICENSE

commit b0311449f92fbeb85e1b53f05bc2f831f4d26d8f
parent fe57630b1dc0c45eb8f6ee8a868a91a07a3cbff0
Author: Randell Jesup <rjesup@mozilla.com>
Date:   Wed,  1 Oct 2025 18:45:56 +0000

Bug 1917975: Read dictionaries from cache for Compression Dictionaries r=necko-reviewers,valentin

Differential Revision: https://phabricator.services.mozilla.com/D258126

Diffstat:
Mnetwerk/cache2/CacheStorage.cpp | 33++++++++++++++++++++-------------
Mnetwerk/cache2/Dictionary.cpp | 198++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------
Mnetwerk/cache2/Dictionary.h | 43+++++++++++++++++++++++++++++--------------
Mnetwerk/cache2/nsICacheStorage.idl | 29++++++++++++++++++++++++++++-
Mnetwerk/protocol/http/HttpBaseChannel.cpp | 8--------
Mnetwerk/protocol/http/nsHttpChannel.cpp | 46++++++++++++++++++++++++++++++++++++++++------
Mnetwerk/protocol/http/nsHttpChannel.h | 4++++
7 files changed, 303 insertions(+), 58 deletions(-)

diff --git a/netwerk/cache2/CacheStorage.cpp b/netwerk/cache2/CacheStorage.cpp @@ -28,6 +28,24 @@ NS_IMETHODIMP CacheStorage::AsyncOpenURI(nsIURI* aURI, const nsACString& aIdExtension, uint32_t aFlags, nsICacheEntryOpenCallback* aCallback) { + NS_ENSURE_ARG(aURI); + + nsresult rv; + + nsCOMPtr<nsIURI> noRefURI; + rv = NS_GetURIWithoutRef(aURI, getter_AddRefs(noRefURI)); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString asciiSpec; + rv = noRefURI->GetAsciiSpec(asciiSpec); + NS_ENSURE_SUCCESS(rv, rv); + + return AsyncOpenURIString(asciiSpec, aIdExtension, aFlags, aCallback); +} + +NS_IMETHODIMP CacheStorage::AsyncOpenURIString( + const nsACString& aURI, const nsACString& aIdExtension, uint32_t aFlags, + nsICacheEntryOpenCallback* aCallback) { if (!CacheStorageService::Self()) return NS_ERROR_NOT_INITIALIZED; if (MOZ_UNLIKELY(!CacheObserver::UseDiskCache()) && mWriteToDisk && @@ -42,22 +60,11 @@ NS_IMETHODIMP CacheStorage::AsyncOpenURI(nsIURI* aURI, return NS_OK; } - NS_ENSURE_ARG(aURI); NS_ENSURE_ARG(aCallback); - nsresult rv; - - nsCOMPtr<nsIURI> noRefURI; - rv = NS_GetURIWithoutRef(aURI, getter_AddRefs(noRefURI)); - NS_ENSURE_SUCCESS(rv, rv); - - nsAutoCString asciiSpec; - rv = noRefURI->GetAsciiSpec(asciiSpec); - NS_ENSURE_SUCCESS(rv, rv); - RefPtr<CacheEntryHandle> entry; - rv = CacheStorageService::Self()->AddStorageEntry( - this, asciiSpec, aIdExtension, aFlags, getter_AddRefs(entry)); + nsresult rv = CacheStorageService::Self()->AddStorageEntry( + this, aURI, aIdExtension, aFlags, getter_AddRefs(entry)); if (NS_FAILED(rv)) { aCallback->OnCacheEntryAvailable(nullptr, false, rv); return NS_OK; diff --git a/netwerk/cache2/Dictionary.cpp b/netwerk/cache2/Dictionary.cpp @@ -8,7 +8,10 @@ #include "Dictionary.h" #include "nsAppDirectoryServiceDefs.h" +#include "nsIAsyncInputStream.h" +#include "nsICacheStorageService.h" #include "nsICacheStorage.h" +#include "nsICacheEntry.h" #include "nsICachingChannel.h" #include "nsICancelable.h" #include "nsIChannel.h" @@ -23,6 +26,7 @@ #include "nsIObserverService.h" #include "nsITimer.h" #include "nsIURI.h" +#include "nsInputStreamPump.h" #include "nsNetUtil.h" #include "nsServiceManagerUtils.h" #include "nsStreamUtils.h" @@ -75,6 +79,9 @@ DictionaryCacheEntry::DictionaryCacheEntry(const nsACString& aURI, } } +NS_IMPL_ISUPPORTS(DictionaryCacheEntry, nsICacheEntryOpenCallback, + nsIStreamListener) + // returns true if the pattern for the dictionary matches the path given bool DictionaryCacheEntry::Match(const nsACString& aFilePath, uint32_t& aLongest) { @@ -113,21 +120,83 @@ bool DictionaryCacheEntry::Match(const nsACString& aFilePath, return false; } -void DictionaryCacheEntry::Prefetch() { - // Start reading the cache entry into memory - // XXX +void DictionaryCacheEntry::InUse() { + mUsers++; + DICTIONARY_LOG(("Dictionary users for %s -- %u Users", mURI.get(), mUsers)); +} + +void DictionaryCacheEntry::UseCompleted() { + MOZ_ASSERT(mUsers > 0); + mUsers--; + // Purge mDictionaryData + if (mUsers == 0) { // XXX perhaps we should hold it for a bit longer? + DICTIONARY_LOG(("Clearing Dictionary data for %s", mURI.get())); + mDictionaryData.clear(); + mDictionaryDataComplete = false; + } else { + DICTIONARY_LOG(("Not clearing Dictionary data for %s -- %u Users", + mURI.get(), mUsers)); + } +} + +// returns aShouldSuspend=true if we should suspend to wait for the prefetch +bool DictionaryCacheEntry::Prefetch(nsILoadContextInfo* aLoadContextInfo, + const std::function<nsresult()>& aFunc) { + DICTIONARY_LOG(("Prefetch for %s", mURI.get())); + // Start reading the cache entry into memory and call completion + // function when done + if (mWaitingPrefetch.IsEmpty()) { + if (mDictionaryData.empty()) { + // We haven't requested it yet from the Cache and don't have it in memory + // already + nsCOMPtr<nsICacheStorageService> cacheStorageService( + components::CacheStorage::Service()); + if (!cacheStorageService) { + return false; + } + nsCOMPtr<nsICacheStorage> cacheStorage; + nsresult rv = cacheStorageService->DiskCacheStorage( + aLoadContextInfo, getter_AddRefs(cacheStorage)); + if (NS_FAILED(rv)) { + return false; + } + // Add before we schedule! + mWaitingPrefetch.AppendElement(aFunc); + cacheStorage->AsyncOpenURIString(mURI, ""_ns, + nsICacheStorage::OPEN_READONLY, this); + DICTIONARY_LOG( + ("Started Prefetch for %s", PromiseFlatCString(mURI).get())); + return true; + } + DICTIONARY_LOG(("Prefetch for %s - already have data in memory (%u users)", + mURI.get(), mUsers)); + return false; + } + DICTIONARY_LOG(("Prefetch for %s - already waiting", mURI.get())); + return true; } void DictionaryCacheEntry::AccumulateHash(const char* aBuf, int32_t aCount) { - if (!mCrypto) { - // If mCrypto is null, and mDictionaryData is set, we've already got the - // data for this dictionary. + MOZ_ASSERT(NS_IsMainThread()); + if (!mHash.IsEmpty()) { if (!mDictionaryData.empty()) { - DICTIONARY_LOG(("%p: Trying to rewrite to Dictionary uri %s, pattern %s", - this, PromiseFlatCString(mURI).get(), - PromiseFlatCString(mPattern).get())); + // We have data from the cache.... but if we change the hash there will + // be problems + // XXX dragons here return; } + // accumulating a new hash when we have an existing? + // XXX probably kill the hash when we get an overwrite; tricky, need to + // handle loading the old one into ram to decompress the new one. Also, + // what if the old one is being used for multiple requests, one of which + // is an overwrite? This is an edge case not discussed in the spec - we + // could separate out a structure for in-flight requests where the data + // would be used from, so the Entry could be overwritten as needed + return; // XXX + } + if (!mCrypto) { + // If mCrypto is null, and mDictionaryData is set, we've already got the + // data for this dictionary. nsresult rv; mCrypto = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); if (NS_WARN_IF(NS_FAILED(rv))) { @@ -156,6 +225,98 @@ void DictionaryCacheEntry::FinishFile() { DICTIONARY_LOG(("Set dictionary hash for %p to %s", this, mHash.get())); mCrypto = nullptr; } + mDictionaryDataComplete = true; + DICTIONARY_LOG(("Unsuspending %zu channels, Dictionary len %zu", + mWaitingPrefetch.Length(), mDictionaryData.length())); + // if we suspended, un-suspend the channel(s) + for (auto& lambda : mWaitingPrefetch) { + (lambda)(); + } + mWaitingPrefetch.Clear(); +} + +//----------------------------------------------------------------------------- +// nsIStreamListener implementation +//----------------------------------------------------------------------------- + +NS_IMETHODIMP +DictionaryCacheEntry::OnStartRequest(nsIRequest* request) { + DICTIONARY_LOG(("DictionaryCacheEntry %s OnStartRequest", mURI.get())); + return NS_OK; +} + +NS_IMETHODIMP +DictionaryCacheEntry::OnDataAvailable(nsIRequest* request, + nsIInputStream* aInputStream, + uint64_t aOffset, uint32_t aCount) { + uint32_t n; + return aInputStream->ReadSegments(&DictionaryCacheEntry::ReadCacheData, this, + aCount, &n); +} + +/* static */ +nsresult DictionaryCacheEntry::ReadCacheData( + nsIInputStream* aInStream, void* aClosure, const char* aFromSegment, + uint32_t aToOffset, uint32_t aCount, uint32_t* aWriteCount) { + DictionaryCacheEntry* self = static_cast<DictionaryCacheEntry*>(aClosure); + + self->AccumulateFile(aFromSegment, aCount); + *aWriteCount = aCount; + return NS_OK; +} + +NS_IMETHODIMP +DictionaryCacheEntry::OnStopRequest(nsIRequest* request, nsresult result) { + DICTIONARY_LOG(("DictionaryCacheEntry %s OnStopRequest", mURI.get())); + if (NS_SUCCEEDED(result)) { + FinishFile(); + } else { + // XXX + } + return NS_OK; +} + +//----------------------------------------------------------------------------- +// nsICacheEntryOpenCallback implementation +//----------------------------------------------------------------------------- +// Note: we don't care if the entry is stale since we're not loading it; we're +// just saying with have this specific set of bits with this hash available +// to use as a dictionary. + +NS_IMETHODIMP +DictionaryCacheEntry::OnCacheEntryCheck(nsICacheEntry* aEntry, + uint32_t* result) { + DICTIONARY_LOG(("OnCacheEntryCheck %s", mURI.get())); + *result = nsICacheEntryOpenCallback::ENTRY_WANTED; + return NS_OK; +} + +NS_IMETHODIMP +DictionaryCacheEntry::OnCacheEntryAvailable(nsICacheEntry* entry, bool isNew, + nsresult status) { + DICTIONARY_LOG(("OnCacheEntryAvailable %s, result %u", + PromiseFlatCString(mURI).get(), (uint32_t)status)); + if (entry) { + nsCOMPtr<nsIInputStream> stream; + entry->OpenInputStream(0, getter_AddRefs(stream)); + if (!stream) { + return NS_OK; + } + + RefPtr<nsInputStreamPump> pump; + nsresult rv = nsInputStreamPump::Create(getter_AddRefs(pump), stream); + if (NS_FAILED(rv)) { + return NS_OK; // just ignore + } + + rv = pump->AsyncRead(this); + if (NS_FAILED(rv)) { + return NS_OK; // just ignore + } + DICTIONARY_LOG(("Waiting for data")); + } + + return NS_OK; } // static @@ -186,14 +347,19 @@ nsresult DictionaryCache::AddEntry(nsIURI* aURI, const nsACString& aKey, auto& list = entry.OrInsertWith([&] { return new DictCacheList; }); for (const auto& dict : *list) { + // We replace on the URI being the same (not id, which is just an + // arbitrary extra id sent back to the server) if (dict->GetURI().Equals(aKey)) { - // We're overwriting an existing entry. It might be the same, of - // course - DICTIONARY_LOG(("Reusing dictionary for %s: %p", + // We're overwriting an existing entry, probably with a new hash. It + // might be the same, of course. + // If there are outstanding requests using this entry, we need + // to keep it alive for the users until they're done, but not use + // it for any new requests. Remove the existing entry from the list + // and replace with a new one; existing requests will have a ref to it. + DICTIONARY_LOG(("Replacing dictionary for %s: %p", PromiseFlatCString(dict->GetURI()).get(), dict)); - dict->Clear(); - *aDictEntry = do_AddRef(dict).take(); - return NS_OK; + dict->remove(); + break; } } // New entry @@ -273,7 +439,7 @@ already_AddRefed<DictionaryCacheEntry> DictionaryCache::GetDictionaryFor( aURI->GetPathQueryRef(path); for (const auto& dict : *(origin->get())) { - if (dict->Valid() && dict->Match(path, longest)) { + if (dict->Match(path, longest)) { result = dict; } } diff --git a/netwerk/cache2/Dictionary.h b/netwerk/cache2/Dictionary.h @@ -40,12 +40,21 @@ namespace net { // When creating an entry from incoming data, we'll create it with no hash // initially until the full data has arrived, then update the Hash. class DictionaryCacheEntry final - : public LinkedListElement<RefPtr<DictionaryCacheEntry>> { + : public LinkedListElement<RefPtr<DictionaryCacheEntry>>, + public nsICacheEntryOpenCallback, + public nsIStreamListener { private: - ~DictionaryCacheEntry() { MOZ_ASSERT(mUsers == 0); } + ~DictionaryCacheEntry() { + MOZ_ASSERT(mUsers == 0); + MOZ_ASSERT(!isInList()); + } public: - NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DictionaryCacheEntry) + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICACHEENTRYOPENCALLBACK + NS_DECL_NSIREQUESTOBSERVER + NS_DECL_NSISTREAMLISTENER + DictionaryCacheEntry(const nsACString& aKey, const nsACString& aPattern, const nsACString& aId, const Maybe<nsCString>& aHash = Nothing()); @@ -53,7 +62,10 @@ class DictionaryCacheEntry final // returns true if the pattern for the dictionary matches the path given bool Match(const nsACString& aFilePath, uint32_t& aLongest); - void Prefetch(); + // Start reading the cache entry into memory and call completion + // function when done + bool Prefetch(nsILoadContextInfo* aLoadContextInfo, + const std::function<nsresult()>& aFunc); const nsACString& GetHash() const { MOZ_ASSERT(NS_IsMainThread()); @@ -65,19 +77,11 @@ class DictionaryCacheEntry final mHash = aHash; } - bool Valid() const { return !mHash.IsEmpty(); } - const nsCString& GetId() const { return mId; } // keep track of requests that may need the data - void InUse() { mUsers++; } - void UseCompleted() { - MOZ_ASSERT(mUsers > 0); - mUsers--; - // Purge mDictionaryData - // XXX Don't clear until we can reload it from disk - // mDictionaryData.clear(); - } + void InUse(); + void UseCompleted(); const nsACString& GetURI() const { return mURI; } @@ -85,6 +89,8 @@ class DictionaryCacheEntry final MOZ_ASSERT(NS_IsMainThread()); mHash.Truncate(0); mDictionaryData.clear(); + mDictionaryDataComplete = false; + MOZ_ASSERT(mWaitingPrefetch.IsEmpty()); } const Vector<uint8_t>& GetDictionary() const { return mDictionaryData; } @@ -100,11 +106,17 @@ class DictionaryCacheEntry final return (uint8_t*)mDictionaryData.begin(); } + bool DictionaryReady() const { return mDictionaryDataComplete; } + size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const { // XXX return mallocSizeOf(this); } + static nsresult ReadCacheData(nsIInputStream* aInStream, void* aClosure, + const char* aFromSegment, uint32_t aToOffset, + uint32_t aCount, uint32_t* aWriteCount); + private: nsCString mURI; // URI (without ref) for the dictionary nsCString mPattern; @@ -122,6 +134,9 @@ class DictionaryCacheEntry final // for accumulating SHA-256 hash values for dictionaries nsCOMPtr<nsICryptoHash> mCrypto; + + // call these when prefetch is complete + nsTArray<std::function<void()>> mWaitingPrefetch; }; // XXX Do we want to pre-read dictionaries into RAM at startup (lazily)? diff --git a/netwerk/cache2/nsICacheStorage.idl b/netwerk/cache2/nsICacheStorage.idl @@ -67,7 +67,7 @@ interface nsICacheStorage : nsISupports * Result is fetched asynchronously via the callback. * * @param aURI - * The URI to search in cache or to open for writting. + * The URI to search in cache or to open for writing. * @param aIdExtension * Any string that will extend (distinguish) the entry. Two entries * with the same aURI but different aIdExtension will be comletely @@ -90,6 +90,33 @@ interface nsICacheStorage : nsISupports in nsICacheEntryOpenCallback aCallback); /** + * Asynchronously opens a cache entry for the specified URI. + * Result is fetched asynchronously via the callback. + * + * @param aURI + * The URI to search in cache or to open for writing. + * @param aIdExtension + * Any string that will extend (distinguish) the entry. Two entries + * with the same aURI but different aIdExtension will be comletely + * different entries. If you don't know what aIdExtension should be + * leave it empty. + * @param aFlags + * OPEN_NORMALLY - open cache entry normally for read and write + * OPEN_TRUNCATE - delete any existing entry before opening it + * OPEN_READONLY - don't create an entry if there is none + * OPEN_PRIORITY - give this request a priority over others + * OPEN_BYPASS_IF_BUSY - backward compatibility only, LOAD_BYPASS_LOCAL_CACHE_IF_BUSY + * CHECK_MULTITHREADED - onCacheEntryCheck may be called on any thread, consumer + * implementation is thread-safe + * @param aCallback + * The consumer that receives the result. + * IMPORTANT: The callback may be called sooner the method returns. + */ + void asyncOpenURIString(in ACString aURI, in ACString aIdExtension, + in uint32_t aFlags, + in nsICacheEntryOpenCallback aCallback); + + /** * Immediately opens a new and empty cache entry in the storage, any existing * entries are immediately doomed. This is similar to the recreate() method * on nsICacheEntry. diff --git a/netwerk/protocol/http/HttpBaseChannel.cpp b/netwerk/protocol/http/HttpBaseChannel.cpp @@ -347,7 +347,6 @@ nsresult HttpBaseChannel::Init(nsIURI* aURI, uint32_t aCaps, // Construct connection info object nsAutoCString host; int32_t port = -1; - bool isHTTPS = isSecureOrTrustworthyURL(mURI); nsresult rv = mURI->GetAsciiHost(host); if (NS_FAILED(rv)) return rv; @@ -375,13 +374,6 @@ nsresult HttpBaseChannel::Init(nsIURI* aURI, uint32_t aCaps, rv = mRequestHead.SetHeader(nsHttp::Host, hostLine); if (NS_FAILED(rv)) return rv; - rv = gHttpHandler->AddAcceptAndDictionaryHeaders(aURI, &mRequestHead, isHTTPS, - mDict); - if (NS_FAILED(rv)) return rv; - if (mDict) { - mDict->InUse(); - } - // Override the Accept header if a specific MediaDocument kind is forced. ExtContentPolicy contentPolicyType = mLoadInfo->GetExternalContentPolicyType(); diff --git a/netwerk/protocol/http/nsHttpChannel.cpp b/netwerk/protocol/http/nsHttpChannel.cpp @@ -505,7 +505,7 @@ nsHttpChannel::~nsHttpChannel() { gHttpHandler->RemoveHttpChannel(mChannelId); } - if (mDict) { + if (mDict && mUsingDictionary) { mDict->UseCompleted(); } } @@ -546,6 +546,29 @@ nsresult nsHttpChannel::Init(nsIURI* uri, uint32_t caps, nsProxyInfo* proxyInfo, nsresult rv = HttpBaseChannel::Init(uri, caps, proxyInfo, proxyResolveFlags, proxyURI, channelId, aLoadInfo); if (NS_FAILED(rv)) return rv; + rv = gHttpHandler->AddAcceptAndDictionaryHeaders(uri, &mRequestHead, + IsHTTPS(), mDict); + if (NS_FAILED(rv)) return rv; + if (mDict) { + // mDict is set if we added Available-Dictionary + mDict->InUse(); + mUsingDictionary = true; + mShouldSuspendForDictionary = mDict->Prefetch( + GetLoadContextInfo(this), [self = RefPtr<nsHttpChannel>(this)]() { + // this is called when the prefetch is complete to + // un-Suspend the channel + MOZ_ASSERT(self->mDict->DictionaryReady()); + if (self->mSuspendedForDictionary) { + LOG( + ("nsHttpChannel::Init [this=%p] Resuming channel suspended for " + "Dictionary", + self.get())); + self->mSuspendedForDictionary = false; + self->Resume(); + } + return NS_OK; + }); + } LOG1(("nsHttpChannel::Init [this=%p]\n", this)); @@ -3560,6 +3583,18 @@ nsresult nsHttpChannel::ContinueProcessNormal(nsresult rv) { } } + // If we don't have the entire dictionary yet, Suspend() the channel + // until the dictionary is in-memory. + if (mDict && mUsingDictionary && mShouldSuspendForDictionary && + !mDict->DictionaryReady()) { + LOG( + ("nsHttpChannel::ContinueProcessNormal [this=%p] Suspending the " + "transaction, waiting for dictionary", + this)); + Suspend(); + mSuspendedForDictionary = true; + } + // We need to do this before CallonStartRequest, since this can modify // the Content-Encoding to remove dcb/dcz (and perhaps others), and // CallOnStartRequest() sends this to the content process. @@ -5974,7 +6009,6 @@ bool nsHttpChannel::ParseDictionary(nsICacheEntry* aEntry, matchIdVal.get(), typeVal.get())); dicts->AddEntry(mURI, key, matchVal, matchIdVal, Some(hash), getter_AddRefs(mDict)); - mDict->InUse(); return true; } return false; @@ -6050,6 +6084,7 @@ nsresult nsHttpChannel::InstallCacheListener(int64_t offset) { nsAutoCString dictionary; Unused << mResponseHead->GetHeader(nsHttp::Use_As_Dictionary, dictionary); if (!dictionary.IsEmpty()) { + // We need to record the hash as we save it mCacheEntry->SetDictionary(mDict); } LOG(("Trading cache input stream for output stream [channel=%p]", this)); @@ -7763,13 +7798,12 @@ nsHttpChannel::GetDictionary(DictionaryCacheEntry** aDictionary) { NS_IMETHODIMP nsHttpChannel::SetDictionary(DictionaryCacheEntry* aDictionary) { - if (mDict) { + if (mDict && mUsingDictionary) { mDict->UseCompleted(); + mUsingDictionary = false; } mDict = aDictionary; - if (aDictionary) { - aDictionary->InUse(); - } + // We're only setting InUse() if we are using it for Available-Dictionary return NS_OK; } diff --git a/netwerk/protocol/http/nsHttpChannel.h b/netwerk/protocol/http/nsHttpChannel.h @@ -885,6 +885,10 @@ class nsHttpChannel final : public HttpBaseChannel, // Used to handle cancellation while suspended waiting for LNA permission bool mWaitingForLNAPermission{false}; + bool mUsingDictionary{false}; // we added Available-Dictionary + bool mShouldSuspendForDictionary{false}; + bool mSuspendedForDictionary{false}; + protected: virtual void DoNotifyListenerCleanup() override;