commit 79e1b8910963fcfe8e4d41bb806ad9b3b8326d26 parent 5619e4c9e6cd936697f6fa85ddeb1afdb15dc57d Author: Atila Butkovits <abutkovits@mozilla.com> Date: Tue, 16 Dec 2025 11:51:38 +0200 Revert "Bug 2006028 - Remove predictor code r=necko-reviewers,firefox-style-system-reviewers,emilio,kershaw" for causing failures at test_basic.js. This reverts commit 117ed7db99075a445deec14d08d4c113810c7d5f. Diffstat:
39 files changed, 4312 insertions(+), 9 deletions(-)
diff --git a/docshell/base/nsDocShell.cpp b/docshell/base/nsDocShell.cpp @@ -138,6 +138,7 @@ #include "nsILoadURIDelegate.h" #include "nsIMultiPartChannel.h" #include "nsINestedURI.h" +#include "nsINetworkPredictor.h" #include "nsINode.h" #include "nsINSSErrorsService.h" #include "nsIObserverService.h" @@ -6604,6 +6605,10 @@ nsresult nsDocShell::EndPageLoad(nsIWebProgress* aProgress, nsContentUtils::eDOM_PROPERTIES, "UnknownProtocolNavigationPrevented", params); } + } else { + // If we have a host + nsCOMPtr<nsILoadInfo> loadInfo = aChannel->LoadInfo(); + PredictorLearnRedirect(url, aChannel, loadInfo->GetOriginAttributes()); } if (hadErrorStatus) { @@ -10078,6 +10083,11 @@ nsresult nsDocShell::InternalLoad(nsDocShellLoadState* aLoadState, OriginAttributes attrs = GetOriginAttributes(); attrs.SetFirstPartyDomain(isTopLevelDoc, aLoadState->URI()); + PredictorLearn(aLoadState->URI(), nullptr, + nsINetworkPredictor::LEARN_LOAD_TOPLEVEL, attrs); + PredictorPredict(aLoadState->URI(), nullptr, + nsINetworkPredictor::PREDICT_LOAD, attrs, nullptr); + nsCOMPtr<nsIRequest> req; rv = DoURILoad(aLoadState, aCacheKey, getter_AddRefs(req)); diff --git a/dom/script/ScriptLoader.cpp b/dom/script/ScriptLoader.cpp @@ -81,6 +81,7 @@ #include "nsIDocShell.h" #include "nsIHttpChannel.h" #include "nsIHttpChannelInternal.h" +#include "nsINetworkPredictor.h" #include "nsIPrincipal.h" #include "nsIScriptContext.h" #include "nsIScriptElement.h" @@ -1004,6 +1005,11 @@ nsresult ScriptLoader::StartLoadInternal( PrepareHttpRequestAndInitiatorType(channel, aRequest, aCharsetForPreload); NS_ENSURE_SUCCESS(rv, rv); + mozilla::net::PredictorLearn( + aRequest->URI(), mDocument->GetDocumentURI(), + nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE, + mDocument->NodePrincipal()->OriginAttributesRef()); + nsCOMPtr<nsIIncrementalStreamLoader> loader; rv = PrepareIncrementalStreamLoader(getter_AddRefs(loader), channel, aRequest); diff --git a/image/imgLoader.cpp b/image/imgLoader.cpp @@ -56,6 +56,7 @@ #include "nsIInterfaceRequestor.h" #include "nsIInterfaceRequestorUtils.h" #include "nsIMemoryReporter.h" +#include "nsINetworkPredictor.h" #include "nsIProgressEventSink.h" #include "nsIProtocolHandler.h" #include "nsImageModule.h" @@ -1887,6 +1888,9 @@ bool imgLoader::ValidateRequestWithNewChannel( // Add the proxy without notifying hvc->AddProxy(req); + mozilla::net::PredictorLearn(aURI, aInitialDocumentURI, + nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE, + aLoadGroup); rv = newChannel->AsyncOpen(listener); if (NS_WARN_IF(NS_FAILED(rv))) { req->CancelAndForgetObserver(rv); @@ -2563,6 +2567,10 @@ nsresult imgLoader::LoadImage( ("[this=%p] imgLoader::LoadImage -- Calling channel->AsyncOpen()\n", this)); + mozilla::net::PredictorLearn(aURI, aInitialDocumentURI, + nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE, + aLoadGroup); + nsresult openRes; openRes = newChannel->AsyncOpen(listener); diff --git a/layout/style/FontFaceSet.cpp b/layout/style/FontFaceSet.cpp @@ -49,6 +49,7 @@ #include "nsIDocShell.h" #include "nsIInputStream.h" #include "nsILoadContext.h" +#include "nsINetworkPredictor.h" #include "nsIPrincipal.h" #include "nsIWebNavigation.h" #include "nsLayoutUtils.h" diff --git a/layout/style/FontFaceSetDocumentImpl.cpp b/layout/style/FontFaceSetDocumentImpl.cpp @@ -20,6 +20,7 @@ #include "nsDOMNavigationTiming.h" #include "nsFontFaceLoader.h" #include "nsIDocShell.h" +#include "nsINetworkPredictor.h" #include "nsISupportsPriority.h" #include "nsIWebNavigation.h" #include "nsPresContext.h" @@ -318,6 +319,9 @@ nsresult FontFaceSetDocumentImpl::StartLoad(gfxUserFontEntry* aUserFontEntry, mLoaders.PutEntry(fontLoader); } + net::PredictorLearn(src.mURI->get(), mDocument->GetDocumentURI(), + nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE, loadGroup); + if (NS_SUCCEEDED(rv)) { fontLoader->StartedLoading(streamLoader); // let the font entry remember the loader, in case we need to cancel it diff --git a/layout/style/FontFaceSetWorkerImpl.cpp b/layout/style/FontFaceSetWorkerImpl.cpp @@ -13,6 +13,7 @@ #include "mozilla/dom/WorkerRunnable.h" #include "nsContentPolicyUtils.h" #include "nsFontFaceLoader.h" +#include "nsINetworkPredictor.h" #include "nsIWebNavigation.h" using namespace mozilla; @@ -283,6 +284,9 @@ nsresult FontFaceSetWorkerImpl::StartLoad(gfxUserFontEntry* aUserFontEntry, mLoaders.PutEntry(fontLoader); + net::PredictorLearn(src.mURI->get(), mWorkerRef->Private()->GetBaseURI(), + nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE, loadGroup); + if (NS_SUCCEEDED(rv)) { fontLoader->StartedLoading(streamLoader); // let the font entry remember the loader, in case we need to cancel it diff --git a/layout/style/Loader.cpp b/layout/style/Loader.cpp @@ -57,6 +57,7 @@ #include "nsICookieJarSettings.h" #include "nsIHttpChannel.h" #include "nsIHttpChannelInternal.h" +#include "nsINetworkPredictor.h" #include "nsIPrincipal.h" #include "nsIScriptError.h" #include "nsIScriptSecurityManager.h" @@ -1153,6 +1154,11 @@ nsresult Loader::LoadSheetSyncInternal(SheetLoadData& aLoadData, // channel to make error recovery simpler. auto streamLoader = MakeRefPtr<StreamLoader>(aLoadData); + if (mDocument) { + net::PredictorLearn(aLoadData.mURI, mDocument->GetDocumentURI(), + nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE, mDocument); + } + // Synchronous loads should only be used internally. Therefore no CORS // policy is needed. nsCOMPtr<nsIChannel> channel; @@ -1438,6 +1444,10 @@ nsresult Loader::LoadSheetAsyncInternal(SheetLoadData& aLoadData, // model is: Necko owns the stream loader, which owns the load data, // which owns us auto streamLoader = MakeRefPtr<StreamLoader>(aLoadData); + if (mDocument) { + net::PredictorLearn(aLoadData.mURI, mDocument->GetDocumentURI(), + nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE, mDocument); + } #ifdef DEBUG { diff --git a/modules/libpref/init/StaticPrefList.yaml b/modules/libpref/init/StaticPrefList.yaml @@ -13876,6 +13876,12 @@ value: false mirror: always +# Enables the predictive service. +- name: network.predictor.enabled + type: bool + value: false + mirror: always + # Disable requests to 0.0.0.0 # See Bug 1889130 - name: network.socket.ip_addr_any.disabled @@ -13922,6 +13928,94 @@ value: false mirror: always +- name: network.predictor.enable-prefetch + type: bool + value: false + mirror: always + +- name: network.predictor.page-degradation.day + type: int32_t + value: 0 + mirror: always +- name: network.predictor.page-degradation.week + type: int32_t + value: 5 + mirror: always +- name: network.predictor.page-degradation.month + type: int32_t + value: 10 + mirror: always +- name: network.predictor.page-degradation.year + type: int32_t + value: 25 + mirror: always +- name: network.predictor.page-degradation.max + type: int32_t + value: 50 + mirror: always + +- name: network.predictor.subresource-degradation.day + type: int32_t + value: 1 + mirror: always +- name: network.predictor.subresource-degradation.week + type: int32_t + value: 10 + mirror: always +- name: network.predictor.subresource-degradation.month + type: int32_t + value: 25 + mirror: always +- name: network.predictor.subresource-degradation.year + type: int32_t + value: 50 + mirror: always +- name: network.predictor.subresource-degradation.max + type: int32_t + value: 100 + mirror: always + +- name: network.predictor.prefetch-rolling-load-count + type: int32_t + value: 10 + mirror: always + +- name: network.predictor.prefetch-min-confidence + type: int32_t + value: 100 + mirror: always +- name: network.predictor.preconnect-min-confidence + type: int32_t + value: 90 + mirror: always +- name: network.predictor.preresolve-min-confidence + type: int32_t + value: 60 + mirror: always + +- name: network.predictor.prefetch-force-valid-for + type: int32_t + value: 10 + mirror: always + +- name: network.predictor.max-resources-per-entry + type: int32_t + value: 100 + mirror: always + +# This is selected in concert with max-resources-per-entry to keep memory +# usage low-ish. The default of the combo of the two is ~50k. +- name: network.predictor.max-uri-length + type: uint32_t + value: 500 + mirror: always + +# A testing flag. +- name: network.predictor.doing-tests + type: bool + value: false + mirror: always + # Indicates whether the `fetchpriority` attribute for elements which support it # (e.g. `<script>`) is enabled. - name: network.fetchpriority.enabled diff --git a/netwerk/base/Predictor.cpp b/netwerk/base/Predictor.cpp @@ -0,0 +1,2412 @@ +/* vim: set ts=2 sts=2 et sw=2: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <algorithm> + +#include "Predictor.h" + +#include "nsAppDirectoryServiceDefs.h" +#include "nsICacheStorage.h" +#include "nsICachingChannel.h" +#include "nsICancelable.h" +#include "nsIChannel.h" +#include "nsContentUtils.h" +#include "nsIDNSService.h" +#include "mozilla/dom/Document.h" +#include "nsIFile.h" +#include "nsIHttpChannel.h" +#include "nsIInputStream.h" +#include "nsILoadContext.h" +#include "nsILoadContextInfo.h" +#include "nsILoadGroup.h" +#include "nsINetworkPredictorVerifier.h" +#include "nsIObserverService.h" +#include "nsISpeculativeConnect.h" +#include "nsITimer.h" +#include "nsIURI.h" +#include "nsNetUtil.h" +#include "nsServiceManagerUtils.h" +#include "nsStreamUtils.h" +#include "nsString.h" +#include "nsThreadUtils.h" +#include "mozilla/Logging.h" + +#include "mozilla/Components.h" +#include "mozilla/OriginAttributes.h" +#include "mozilla/Preferences.h" +#include "mozilla/SchedulerGroup.h" +#include "mozilla/StaticPrefs_network.h" +#include "mozilla/glean/NetwerkMetrics.h" + +#include "mozilla/net/NeckoCommon.h" +#include "mozilla/net/NeckoParent.h" + +#include "LoadContextInfo.h" +#include "mozilla/ipc/URIUtils.h" +#include "SerializedLoadContext.h" +#include "mozilla/net/NeckoChild.h" + +#include "mozilla/dom/ContentParent.h" +#include "mozilla/ClearOnShutdown.h" + +#include "CacheControlParser.h" +#include "ReferrerInfo.h" + +using namespace mozilla; + +namespace mozilla { +namespace net { + +Predictor* Predictor::sSelf = nullptr; + +static LazyLogModule gPredictorLog("NetworkPredictor"); + +#define PREDICTOR_LOG(args) \ + MOZ_LOG(gPredictorLog, mozilla::LogLevel::Debug, args) + +#define NOW_IN_SECONDS() static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC) + +// All these time values are in sec +static const uint32_t ONE_DAY = 86400U; +static const uint32_t ONE_WEEK = 7U * ONE_DAY; +static const uint32_t ONE_MONTH = 30U * ONE_DAY; +static const uint32_t ONE_YEAR = 365U * ONE_DAY; + +// Version of metadata entries we expect +static const uint32_t METADATA_VERSION = 1; + +// Flags available in entries +// FLAG_PREFETCHABLE - we have determined that this item is eligible for +// prefetch +static const uint32_t FLAG_PREFETCHABLE = 1 << 0; + +// We save 12 bits in the "flags" section of our metadata for actual flags, the +// rest are to keep track of a rolling count of which loads a resource has been +// used on to determine if we can prefetch that resource or not; +static const uint8_t kRollingLoadOffset = 12; +static const int32_t kMaxPrefetchRollingLoadCount = 20; +static const uint32_t kFlagsMask = ((1 << kRollingLoadOffset) - 1); + +// ID Extensions for cache entries +#define PREDICTOR_ORIGIN_EXTENSION "predictor-origin" + +// Get the full origin (scheme, host, port) out of a URI (maybe should be part +// of nsIURI instead?) +static nsresult ExtractOrigin(nsIURI* uri, nsIURI** originUri) { + nsAutoCString s; + nsresult rv = nsContentUtils::GetWebExposedOriginSerialization(uri, s); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_NewURI(originUri, s); +} + +// All URIs we get passed *must* be http or https if they're not null. This +// helps ensure that. +static bool IsNullOrHttp(nsIURI* uri) { + if (!uri) { + return true; + } + + return SchemeIsHttpOrHttps(uri); +} + +// Listener for the speculative DNS requests we'll fire off, which just ignores +// the result (since we're just trying to warm the cache). This also exists to +// reduce round-trips to the main thread, by being something threadsafe the +// Predictor can use. + +NS_IMPL_ISUPPORTS(Predictor::DNSListener, nsIDNSListener); + +NS_IMETHODIMP +Predictor::DNSListener::OnLookupComplete(nsICancelable* request, + nsIDNSRecord* rec, nsresult status) { + return NS_OK; +} + +// Class to proxy important information from the initial predictor call through +// the cache API and back into the internals of the predictor. We can't use the +// predictor itself, as it may have multiple actions in-flight, and each action +// has different parameters. +NS_IMPL_ISUPPORTS(Predictor::Action, nsICacheEntryOpenCallback); + +Predictor::Action::Action(bool fullUri, bool predict, Predictor::Reason reason, + nsIURI* targetURI, nsIURI* sourceURI, + nsINetworkPredictorVerifier* verifier, + Predictor* predictor) + : mFullUri(fullUri), + mPredict(predict), + mTargetURI(targetURI), + mSourceURI(sourceURI), + mVerifier(verifier), + mStackCount(0), + mPredictor(predictor) { + mStartTime = TimeStamp::Now(); + if (mPredict) { + mPredictReason = reason.mPredict; + } else { + mLearnReason = reason.mLearn; + } +} + +Predictor::Action::Action(bool fullUri, bool predict, Predictor::Reason reason, + nsIURI* targetURI, nsIURI* sourceURI, + nsINetworkPredictorVerifier* verifier, + Predictor* predictor, uint8_t stackCount) + : mFullUri(fullUri), + mPredict(predict), + mTargetURI(targetURI), + mSourceURI(sourceURI), + mVerifier(verifier), + mStackCount(stackCount), + mPredictor(predictor) { + mStartTime = TimeStamp::Now(); + if (mPredict) { + mPredictReason = reason.mPredict; + } else { + mLearnReason = reason.mLearn; + } +} + +NS_IMETHODIMP +Predictor::Action::OnCacheEntryCheck(nsICacheEntry* entry, uint32_t* result) { + *result = nsICacheEntryOpenCallback::ENTRY_WANTED; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Action::OnCacheEntryAvailable(nsICacheEntry* entry, bool isNew, + nsresult result) { + MOZ_ASSERT(NS_IsMainThread(), "Got cache entry off main thread!"); + + nsAutoCString targetURI, sourceURI; + if (!mTargetURI) { + return NS_ERROR_UNEXPECTED; + } + mTargetURI->GetAsciiSpec(targetURI); + if (mSourceURI) { + mSourceURI->GetAsciiSpec(sourceURI); + } + PREDICTOR_LOG( + ("OnCacheEntryAvailable %p called. entry=%p mFullUri=%d mPredict=%d " + "mPredictReason=%d mLearnReason=%d mTargetURI=%s " + "mSourceURI=%s mStackCount=%d isNew=%d result=0x%08" PRIx32, + this, entry, mFullUri, mPredict, mPredictReason, mLearnReason, + targetURI.get(), sourceURI.get(), mStackCount, isNew, + static_cast<uint32_t>(result))); + if (NS_FAILED(result)) { + PREDICTOR_LOG( + ("OnCacheEntryAvailable %p FAILED to get cache entry (0x%08" PRIX32 + "). Aborting.", + this, static_cast<uint32_t>(result))); + return NS_OK; + } + glean::predictor::wait_time.AccumulateRawDuration(TimeStamp::Now() - + mStartTime); + if (mPredict) { + bool predicted = + mPredictor->PredictInternal(mPredictReason, entry, isNew, mFullUri, + mTargetURI, mVerifier, mStackCount); + glean::predictor::predict_work_time.AccumulateRawDuration(TimeStamp::Now() - + mStartTime); + if (predicted) { + glean::predictor::predict_time_to_action.AccumulateRawDuration( + TimeStamp::Now() - mStartTime); + } else { + glean::predictor::predict_time_to_inaction.AccumulateRawDuration( + TimeStamp::Now() - mStartTime); + } + } else { + mPredictor->LearnInternal(mLearnReason, entry, isNew, mFullUri, mTargetURI, + mSourceURI); + glean::predictor::learn_work_time.AccumulateRawDuration(TimeStamp::Now() - + mStartTime); + } + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(Predictor, nsINetworkPredictor, nsIObserver, + nsISpeculativeConnectionOverrider, nsIInterfaceRequestor, + nsICacheEntryMetaDataVisitor, nsINetworkPredictorVerifier) + +Predictor::Predictor() + +{ + MOZ_ASSERT(!sSelf, "multiple Predictor instances!"); + sSelf = this; +} + +Predictor::~Predictor() { + if (mInitialized) Shutdown(); + + sSelf = nullptr; +} + +// Predictor::nsIObserver + +nsresult Predictor::InstallObserver() { + MOZ_ASSERT(NS_IsMainThread(), "Installing observer off main thread"); + + nsresult rv = NS_OK; + nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService(); + if (!obs) { + return NS_ERROR_NOT_AVAILABLE; + } + + rv = obs->AddObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID, false); + NS_ENSURE_SUCCESS(rv, rv); + + return rv; +} + +void Predictor::RemoveObserver() { + MOZ_ASSERT(NS_IsMainThread(), "Removing observer off main thread"); + + nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService(); + if (obs) { + obs->RemoveObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID); + } +} + +NS_IMETHODIMP +Predictor::Observe(nsISupports* subject, const char* topic, + const char16_t* data_unicode) { + nsresult rv = NS_OK; + MOZ_ASSERT(NS_IsMainThread(), + "Predictor observing something off main thread!"); + + if (!strcmp(NS_XPCOM_SHUTDOWN_OBSERVER_ID, topic)) { + Shutdown(); + } + + return rv; +} + +// Predictor::nsISpeculativeConnectionOverrider + +NS_IMETHODIMP +Predictor::GetIgnoreIdle(bool* ignoreIdle) { + *ignoreIdle = true; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::GetParallelSpeculativeConnectLimit( + uint32_t* parallelSpeculativeConnectLimit) { + *parallelSpeculativeConnectLimit = 6; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::GetIsFromPredictor(bool* isFromPredictor) { + *isFromPredictor = true; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::GetAllow1918(bool* allow1918) { + *allow1918 = false; + return NS_OK; +} + +// Predictor::nsIInterfaceRequestor + +NS_IMETHODIMP +Predictor::GetInterface(const nsIID& iid, void** result) { + return QueryInterface(iid, result); +} + +// Predictor::nsICacheEntryMetaDataVisitor + +#define SEEN_META_DATA "predictor::seen" +#define RESOURCE_META_DATA "predictor::resource-count" +#define META_DATA_PREFIX "predictor::" + +static bool IsURIMetadataElement(const char* key) { + return StringBeginsWith(nsDependentCString(key), + nsLiteralCString(META_DATA_PREFIX)) && + !nsLiteralCString(SEEN_META_DATA).Equals(key) && + !nsLiteralCString(RESOURCE_META_DATA).Equals(key); +} + +nsresult Predictor::OnMetaDataElement(const char* asciiKey, + const char* asciiValue) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsURIMetadataElement(asciiKey)) { + // This isn't a bit of metadata we care about + return NS_OK; + } + + nsCString key, value; + key.AssignASCII(asciiKey); + value.AssignASCII(asciiValue); + mKeysToOperateOn.AppendElement(key); + mValuesToOperateOn.AppendElement(value); + + return NS_OK; +} + +// Predictor::nsINetworkPredictor + +nsresult Predictor::Init() { + MOZ_DIAGNOSTIC_ASSERT(!IsNeckoChild()); + + if (!NS_IsMainThread()) { + MOZ_ASSERT(false, "Predictor::Init called off the main thread!"); + return NS_ERROR_UNEXPECTED; + } + + nsresult rv = NS_OK; + + rv = InstallObserver(); + NS_ENSURE_SUCCESS(rv, rv); + + mLastStartupTime = mStartupTime = NOW_IN_SECONDS(); + + if (!mDNSListener) { + mDNSListener = new DNSListener(); + } + + mCacheStorageService = mozilla::components::CacheStorage::Service(&rv); + NS_ENSURE_SUCCESS(rv, rv); + + mSpeculativeService = mozilla::components::IO::Service(&rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = NS_NewURI(getter_AddRefs(mStartupURI), "predictor://startup"); + NS_ENSURE_SUCCESS(rv, rv); + + mDnsService = mozilla::components::DNS::Service(&rv); + NS_ENSURE_SUCCESS(rv, rv); + + mInitialized = true; + + return rv; +} + +namespace { +class PredictorLearnRunnable final : public Runnable { + public: + PredictorLearnRunnable(nsIURI* targetURI, nsIURI* sourceURI, + PredictorLearnReason reason, + const OriginAttributes& oa) + : Runnable("PredictorLearnRunnable"), + mTargetURI(targetURI), + mSourceURI(sourceURI), + mReason(reason), + mOA(oa) { + MOZ_DIAGNOSTIC_ASSERT(targetURI, "Must have a target URI"); + } + + ~PredictorLearnRunnable() = default; + + NS_IMETHOD Run() override { + if (!gNeckoChild) { + // This may have gone away between when this runnable was dispatched and + // when it actually runs, so let's be safe here, even though we asserted + // earlier. + PREDICTOR_LOG(("predictor::learn (async) gNeckoChild went away")); + return NS_OK; + } + + PREDICTOR_LOG(("predictor::learn (async) forwarding to parent")); + gNeckoChild->SendPredLearn(mTargetURI, mSourceURI, mReason, mOA); + + return NS_OK; + } + + private: + nsCOMPtr<nsIURI> mTargetURI; + nsCOMPtr<nsIURI> mSourceURI; + PredictorLearnReason mReason; + const OriginAttributes mOA; +}; + +} // namespace + +void Predictor::Shutdown() { + if (!NS_IsMainThread()) { + MOZ_ASSERT(false, "Predictor::Shutdown called off the main thread!"); + return; + } + + RemoveObserver(); + + mInitialized = false; +} + +nsresult Predictor::Create(const nsIID& aIID, void** aResult) { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv; + + RefPtr<Predictor> svc = new Predictor(); + if (IsNeckoChild()) { + NeckoChild::InitNeckoChild(); + + // Child threads only need to be call into the public interface methods + // so we don't bother with initialization + return svc->QueryInterface(aIID, aResult); + } + + rv = svc->Init(); + if (NS_FAILED(rv)) { + PREDICTOR_LOG(("Failed to initialize predictor, predictor will be a noop")); + } + + // We treat init failure the same as the service being disabled, since this + // is all an optimization anyway. No need to freak people out. That's why we + // gladly continue on QI'ing here. + rv = svc->QueryInterface(aIID, aResult); + + return rv; +} + +NS_IMETHODIMP +Predictor::Predict(nsIURI* targetURI, nsIURI* sourceURI, + PredictorPredictReason reason, + JS::Handle<JS::Value> originAttributes, + nsINetworkPredictorVerifier* verifier, JSContext* aCx) { + OriginAttributes attrs; + + if (!originAttributes.isObject() || !attrs.Init(aCx, originAttributes)) { + return NS_ERROR_INVALID_ARG; + } + + return PredictNative(targetURI, sourceURI, reason, attrs, verifier); +} + +// Called from the main thread to initiate predictive actions +NS_IMETHODIMP +Predictor::PredictNative(nsIURI* targetURI, nsIURI* sourceURI, + PredictorPredictReason reason, + const OriginAttributes& originAttributes, + nsINetworkPredictorVerifier* verifier) { + MOZ_ASSERT(NS_IsMainThread(), + "Predictor interface methods must be called on the main thread"); + + PREDICTOR_LOG(("Predictor::Predict")); + + if (!StaticPrefs::network_predictor_enabled()) { + PREDICTOR_LOG((" not enabled")); + return NS_OK; + } + + if (IsNeckoChild()) { + if (!gNeckoChild) { + return NS_ERROR_FAILURE; + } + + PREDICTOR_LOG((" called on child process")); + // If two different threads are predicting concurently, this will be + // overwritten. Thankfully, we only use this in tests, which will + // overwrite mVerifier perhaps multiple times for each individual test; + // however, within each test, the multiple predict calls should have the + // same verifier. + if (verifier) { + PREDICTOR_LOG((" was given a verifier")); + mChildVerifier = verifier; + } + PREDICTOR_LOG((" forwarding to parent process")); + gNeckoChild->SendPredPredict(targetURI, sourceURI, reason, originAttributes, + verifier); + return NS_OK; + } + + PREDICTOR_LOG((" called on parent process")); + + if (!mInitialized) { + PREDICTOR_LOG((" not initialized")); + return NS_OK; + } + + if (originAttributes.IsPrivateBrowsing()) { + // Don't want to do anything in PB mode + PREDICTOR_LOG((" in PB mode")); + return NS_OK; + } + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + // Nothing we can do for non-HTTP[S] schemes + PREDICTOR_LOG((" got non-http[s] URI")); + return NS_OK; + } + + // Ensure we've been given the appropriate arguments for the kind of + // prediction we're being asked to do + nsCOMPtr<nsIURI> uriKey = targetURI; + nsCOMPtr<nsIURI> originKey; + switch (reason) { + case nsINetworkPredictor::PREDICT_LOAD: + if (!targetURI || sourceURI) { + PREDICTOR_LOG((" load invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + break; + case nsINetworkPredictor::PREDICT_STARTUP: + if (targetURI || sourceURI) { + PREDICTOR_LOG((" startup invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + uriKey = mStartupURI; + originKey = mStartupURI; + break; + default: + PREDICTOR_LOG((" invalid reason")); + return NS_ERROR_INVALID_ARG; + } + + Predictor::Reason argReason{}; + argReason.mPredict = reason; + + // First we open the regular cache entry, to ensure we don't gum up the works + // waiting on the less-important predictor-only cache entry + RefPtr<Predictor::Action> uriAction = new Predictor::Action( + Predictor::Action::IS_FULL_URI, Predictor::Action::DO_PREDICT, argReason, + uriKey, nullptr, verifier, this); + nsAutoCString uriKeyStr; + uriKey->GetAsciiSpec(uriKeyStr); + PREDICTOR_LOG((" Predict uri=%s reason=%d action=%p", uriKeyStr.get(), + reason, uriAction.get())); + + nsCOMPtr<nsICacheStorage> cacheDiskStorage; + + RefPtr<LoadContextInfo> lci = new LoadContextInfo(false, originAttributes); + + nsresult rv = mCacheStorageService->DiskCacheStorage( + lci, getter_AddRefs(cacheDiskStorage)); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t openFlags = + nsICacheStorage::OPEN_READONLY | nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::OPEN_PRIORITY | nsICacheStorage::CHECK_MULTITHREADED; + cacheDiskStorage->AsyncOpenURI(uriKey, ""_ns, openFlags, uriAction); + + // Now we do the origin-only (and therefore predictor-only) entry + nsCOMPtr<nsIURI> targetOrigin; + rv = ExtractOrigin(uriKey, getter_AddRefs(targetOrigin)); + NS_ENSURE_SUCCESS(rv, rv); + if (!originKey) { + originKey = targetOrigin; + } + + RefPtr<Predictor::Action> originAction = new Predictor::Action( + Predictor::Action::IS_ORIGIN, Predictor::Action::DO_PREDICT, argReason, + targetOrigin, nullptr, verifier, this); + nsAutoCString originKeyStr; + originKey->GetAsciiSpec(originKeyStr); + PREDICTOR_LOG((" Predict origin=%s reason=%d action=%p", + originKeyStr.get(), reason, originAction.get())); + openFlags = nsICacheStorage::OPEN_READONLY | nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::CHECK_MULTITHREADED; + cacheDiskStorage->AsyncOpenURI(originKey, + nsLiteralCString(PREDICTOR_ORIGIN_EXTENSION), + openFlags, originAction); + + PREDICTOR_LOG((" predict returning")); + return NS_OK; +} + +bool Predictor::PredictInternal(PredictorPredictReason reason, + nsICacheEntry* entry, bool isNew, bool fullUri, + nsIURI* targetURI, + nsINetworkPredictorVerifier* verifier, + uint8_t stackCount) { + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::PredictInternal")); + bool rv = false; + + nsCOMPtr<nsILoadContextInfo> lci; + entry->GetLoadContextInfo(getter_AddRefs(lci)); + + if (!lci) { + return rv; + } + + if (reason == nsINetworkPredictor::PREDICT_LOAD) { + MaybeLearnForStartup(targetURI, fullUri, *lci->OriginAttributesPtr()); + } + + if (isNew) { + // nothing else we can do here + PREDICTOR_LOG((" new entry")); + return rv; + } + + switch (reason) { + case nsINetworkPredictor::PREDICT_LOAD: + rv = PredictForPageload(entry, targetURI, stackCount, fullUri, verifier); + break; + case nsINetworkPredictor::PREDICT_STARTUP: + rv = PredictForStartup(entry, fullUri, verifier); + break; + default: + PREDICTOR_LOG((" invalid reason")); + MOZ_ASSERT(false, "Got unexpected value for prediction reason"); + } + + return rv; +} + +// This is the driver for prediction based on a new pageload. +static const uint8_t MAX_PAGELOAD_DEPTH = 10; +bool Predictor::PredictForPageload(nsICacheEntry* entry, nsIURI* targetURI, + uint8_t stackCount, bool fullUri, + nsINetworkPredictorVerifier* verifier) { + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::PredictForPageload")); + + if (stackCount > MAX_PAGELOAD_DEPTH) { + PREDICTOR_LOG((" exceeded recursion depth!")); + return false; + } + + uint32_t lastLoad; + nsresult rv = entry->GetLastFetched(&lastLoad); + NS_ENSURE_SUCCESS(rv, false); + + int32_t globalDegradation = CalculateGlobalDegradation(lastLoad); + PREDICTOR_LOG((" globalDegradation = %d", globalDegradation)); + + uint32_t loadCount; + rv = entry->GetFetchCount(&loadCount); + NS_ENSURE_SUCCESS(rv, false); + + nsCOMPtr<nsILoadContextInfo> lci; + + rv = entry->GetLoadContextInfo(getter_AddRefs(lci)); + NS_ENSURE_SUCCESS(rv, false); + + nsCOMPtr<nsIURI> redirectURI; + if (WouldRedirect(entry, loadCount, lastLoad, globalDegradation, + getter_AddRefs(redirectURI))) { + mPreconnects.AppendElement(redirectURI); + Predictor::Reason reason{}; + reason.mPredict = nsINetworkPredictor::PREDICT_LOAD; + RefPtr<Predictor::Action> redirectAction = new Predictor::Action( + Predictor::Action::IS_FULL_URI, Predictor::Action::DO_PREDICT, reason, + redirectURI, nullptr, verifier, this, stackCount + 1); + nsAutoCString redirectUriString; + redirectURI->GetAsciiSpec(redirectUriString); + + nsCOMPtr<nsICacheStorage> cacheDiskStorage; + + rv = mCacheStorageService->DiskCacheStorage( + lci, getter_AddRefs(cacheDiskStorage)); + NS_ENSURE_SUCCESS(rv, false); + + PREDICTOR_LOG((" Predict redirect uri=%s action=%p", + redirectUriString.get(), redirectAction.get())); + uint32_t openFlags = + nsICacheStorage::OPEN_READONLY | nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::OPEN_PRIORITY | nsICacheStorage::CHECK_MULTITHREADED; + cacheDiskStorage->AsyncOpenURI(redirectURI, ""_ns, openFlags, + redirectAction); + return RunPredictions(nullptr, *lci->OriginAttributesPtr(), verifier); + } + + CalculatePredictions(entry, targetURI, lastLoad, loadCount, globalDegradation, + fullUri); + + return RunPredictions(targetURI, *lci->OriginAttributesPtr(), verifier); +} + +// This is the driver for predicting at browser startup time based on pages that +// have previously been loaded close to startup. +bool Predictor::PredictForStartup(nsICacheEntry* entry, bool fullUri, + nsINetworkPredictorVerifier* verifier) { + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::PredictForStartup")); + + nsCOMPtr<nsILoadContextInfo> lci; + + nsresult rv = entry->GetLoadContextInfo(getter_AddRefs(lci)); + NS_ENSURE_SUCCESS(rv, false); + + int32_t globalDegradation = CalculateGlobalDegradation(mLastStartupTime); + CalculatePredictions(entry, nullptr, mLastStartupTime, mStartupCount, + globalDegradation, fullUri); + return RunPredictions(nullptr, *lci->OriginAttributesPtr(), verifier); +} + +// This calculates how much to degrade our confidence in our data based on +// the last time this top-level resource was loaded. This "global degradation" +// applies to *all* subresources we have associated with the top-level +// resource. This will be in addition to any reduction in confidence we have +// associated with a particular subresource. +int32_t Predictor::CalculateGlobalDegradation(uint32_t lastLoad) { + MOZ_ASSERT(NS_IsMainThread()); + + int32_t globalDegradation; + uint32_t delta = NOW_IN_SECONDS() - lastLoad; + if (delta < ONE_DAY) { + globalDegradation = StaticPrefs::network_predictor_page_degradation_day(); + } else if (delta < ONE_WEEK) { + globalDegradation = StaticPrefs::network_predictor_page_degradation_week(); + } else if (delta < ONE_MONTH) { + globalDegradation = StaticPrefs::network_predictor_page_degradation_month(); + } else if (delta < ONE_YEAR) { + globalDegradation = StaticPrefs::network_predictor_page_degradation_year(); + } else { + globalDegradation = StaticPrefs::network_predictor_page_degradation_max(); + } + + glean::predictor::global_degradation.AccumulateSingleSample( + globalDegradation); + return globalDegradation; +} + +// This calculates our overall confidence that a particular subresource will be +// loaded as part of a top-level load. +// @param hitCount - the number of times we have loaded this subresource as part +// of this top-level load +// @param hitsPossible - the number of times we have performed this top-level +// load +// @param lastHit - the timestamp of the last time we loaded this subresource as +// part of this top-level load +// @param lastPossible - the timestamp of the last time we performed this +// top-level load +// @param globalDegradation - the degradation for this top-level load as +// determined by CalculateGlobalDegradation +int32_t Predictor::CalculateConfidence(uint32_t hitCount, uint32_t hitsPossible, + uint32_t lastHit, uint32_t lastPossible, + int32_t globalDegradation) { + MOZ_ASSERT(NS_IsMainThread()); + + uint32_t predictionsCalculated = 1; + + if (!hitsPossible) { + glean::predictor::predictions_calculated.AccumulateSingleSample( + predictionsCalculated); + return 0; + } + + int32_t baseConfidence = (hitCount * 100) / hitsPossible; + int32_t maxConfidence = 100; + int32_t confidenceDegradation = 0; + + if (lastHit < lastPossible) { + // We didn't load this subresource the last time this top-level load was + // performed, so let's not bother preconnecting (at the very least). + maxConfidence = + StaticPrefs::network_predictor_preconnect_min_confidence() - 1; + + // Now calculate how much we want to degrade our confidence based on how + // long it's been between the last time we did this top-level load and the + // last time this top-level load included this subresource. + PRTime delta = lastPossible - lastHit; + if (delta == 0) { + confidenceDegradation = 0; + } else if (delta < ONE_DAY) { + confidenceDegradation = + StaticPrefs::network_predictor_subresource_degradation_day(); + } else if (delta < ONE_WEEK) { + confidenceDegradation = + StaticPrefs::network_predictor_subresource_degradation_week(); + } else if (delta < ONE_MONTH) { + confidenceDegradation = + StaticPrefs::network_predictor_subresource_degradation_month(); + } else if (delta < ONE_YEAR) { + confidenceDegradation = + StaticPrefs::network_predictor_subresource_degradation_year(); + } else { + confidenceDegradation = + StaticPrefs::network_predictor_subresource_degradation_max(); + maxConfidence = 0; + } + } + + // Calculate our confidence and clamp it to between 0 and maxConfidence + // (<= 100) + int32_t confidence = + baseConfidence - confidenceDegradation - globalDegradation; + confidence = std::max(confidence, 0); + confidence = std::min(confidence, maxConfidence); + + glean::predictor::base_confidence.AccumulateSingleSample(baseConfidence); + glean::predictor::subresource_degradation.AccumulateSingleSample( + confidenceDegradation); + glean::predictor::confidence.AccumulateSingleSample(confidence); + glean::predictor::predictions_calculated.AccumulateSingleSample( + predictionsCalculated); + return confidence; +} + +static void MakeMetadataEntry(const uint32_t hitCount, const uint32_t lastHit, + const uint32_t flags, nsCString& newValue) { + newValue.Truncate(); + newValue.AppendInt(METADATA_VERSION); + newValue.Append(','); + newValue.AppendInt(hitCount); + newValue.Append(','); + newValue.AppendInt(lastHit); + newValue.Append(','); + newValue.AppendInt(flags); +} + +// On every page load, the rolling window gets shifted by one bit, leaving the +// lowest bit at 0, to indicate that the subresource in question has not been +// seen on the most recent page load. If, at some point later during the page +// load, the subresource is seen again, we will then set the lowest bit to 1. +// This is how we keep track of how many of the last n pageloads (for n <= 20) a +// particular subresource has been seen. The rolling window is kept in the upper +// 20 bits of the flags element of the metadata. This saves 12 bits for regular +// old flags. +void Predictor::UpdateRollingLoadCount(nsICacheEntry* entry, + const uint32_t flags, const char* key, + const uint32_t hitCount, + const uint32_t lastHit) { + // Extract just the rolling load count from the flags, shift it to clear the + // lowest bit, and put the new value with the existing flags. + uint32_t rollingLoadCount = flags & ~kFlagsMask; + rollingLoadCount <<= 1; + uint32_t newFlags = (flags & kFlagsMask) | rollingLoadCount; + + // Finally, update the metadata on the cache entry. + nsAutoCString newValue; + MakeMetadataEntry(hitCount, lastHit, newFlags, newValue); + entry->SetMetaDataElement(key, newValue.BeginReading()); +} + +uint32_t Predictor::ClampedPrefetchRollingLoadCount() { + int32_t n = StaticPrefs::network_predictor_prefetch_rolling_load_count(); + if (n < 0) { + return 0; + } + if (n > kMaxPrefetchRollingLoadCount) { + return kMaxPrefetchRollingLoadCount; + } + return n; +} + +void Predictor::CalculatePredictions(nsICacheEntry* entry, nsIURI* referrer, + uint32_t lastLoad, uint32_t loadCount, + int32_t globalDegradation, bool fullUri) { + MOZ_ASSERT(NS_IsMainThread()); + + // Since the visitor gets called under a cache lock, all we do there is get + // copies of the keys/values we care about, and then do the real work here + entry->VisitMetaData(this); + nsTArray<nsCString> keysToOperateOn = std::move(mKeysToOperateOn), + valuesToOperateOn = std::move(mValuesToOperateOn); + + MOZ_ASSERT(keysToOperateOn.Length() == valuesToOperateOn.Length()); + for (size_t i = 0; i < keysToOperateOn.Length(); ++i) { + const char* key = keysToOperateOn[i].BeginReading(); + const char* value = valuesToOperateOn[i].BeginReading(); + + nsCString uri; + uint32_t hitCount, lastHit, flags; + if (!ParseMetaDataEntry(key, value, uri, hitCount, lastHit, flags)) { + // This failed, get rid of it so we don't waste space + entry->SetMetaDataElement(key, nullptr); + continue; + } + + int32_t confidence = CalculateConfidence(hitCount, loadCount, lastHit, + lastLoad, globalDegradation); + if (fullUri) { + UpdateRollingLoadCount(entry, flags, key, hitCount, lastHit); + } + PREDICTOR_LOG(("CalculatePredictions key=%s value=%s confidence=%d", key, + value, confidence)); + PrefetchIgnoreReason reason = PREFETCH_OK; + if (!fullUri) { + // Not full URI - don't prefetch! No sense in it! + PREDICTOR_LOG((" forcing non-cacheability - not full URI")); + if (flags & FLAG_PREFETCHABLE) { + // This only applies if we had somehow otherwise marked this + // prefetchable. + reason = NOT_FULL_URI; + } + flags &= ~FLAG_PREFETCHABLE; + } else if (!referrer) { + // No referrer means we can't prefetch, so pretend it's non-cacheable, + // no matter what. + PREDICTOR_LOG((" forcing non-cacheability - no referrer")); + if (flags & FLAG_PREFETCHABLE) { + // This only applies if we had somehow otherwise marked this + // prefetchable. + reason = NO_REFERRER; + } + flags &= ~FLAG_PREFETCHABLE; + } else { + uint32_t expectedRollingLoadCount = + (1 << ClampedPrefetchRollingLoadCount()) - 1; + expectedRollingLoadCount <<= kRollingLoadOffset; + if ((flags & expectedRollingLoadCount) != expectedRollingLoadCount) { + PREDICTOR_LOG((" forcing non-cacheability - missed a load")); + if (flags & FLAG_PREFETCHABLE) { + // This only applies if we had somehow otherwise marked this + // prefetchable. + reason = MISSED_A_LOAD; + } + flags &= ~FLAG_PREFETCHABLE; + } + } + + PREDICTOR_LOG((" setting up prediction")); + SetupPrediction(confidence, flags, uri, reason); + } +} + +// (Maybe) adds a predictive action to the prediction runner, based on our +// calculated confidence for the subresource in question. +void Predictor::SetupPrediction(int32_t confidence, uint32_t flags, + const nsCString& uri, + PrefetchIgnoreReason earlyReason) { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv = NS_OK; + PREDICTOR_LOG( + ("SetupPrediction enable-prefetch=%d prefetch-min-confidence=%d " + "preconnect-min-confidence=%d preresolve-min-confidence=%d " + "flags=%d confidence=%d uri=%s", + StaticPrefs::network_predictor_enable_prefetch(), + StaticPrefs::network_predictor_prefetch_min_confidence(), + StaticPrefs::network_predictor_preconnect_min_confidence(), + StaticPrefs::network_predictor_preresolve_min_confidence(), flags, + confidence, uri.get())); + + bool prefetchOk = !!(flags & FLAG_PREFETCHABLE); + PrefetchIgnoreReason reason = earlyReason; + if (prefetchOk && !StaticPrefs::network_predictor_enable_prefetch()) { + prefetchOk = false; + reason = PREFETCH_DISABLED; + } else if (prefetchOk && !ClampedPrefetchRollingLoadCount() && + confidence < + StaticPrefs::network_predictor_prefetch_min_confidence()) { + prefetchOk = false; + if (!ClampedPrefetchRollingLoadCount()) { + reason = PREFETCH_DISABLED_VIA_COUNT; + } else { + reason = CONFIDENCE_TOO_LOW; + } + } + + // prefetchOk == false and reason == PREFETCH_OK indicates that the reason + // we aren't prefetching this item is because it was marked un-prefetchable in + // our metadata. We already have separate telemetry on that decision, so we + // aren't going to accumulate more here. Right now we only care about why + // something we had marked prefetchable isn't being prefetched. + if (!prefetchOk && reason != PREFETCH_OK) { + glean::predictor::prefetch_ignore_reason.AccumulateSingleSample(reason); + } + + if (prefetchOk) { + nsCOMPtr<nsIURI> prefetchURI; + rv = NS_NewURI(getter_AddRefs(prefetchURI), uri); + if (NS_SUCCEEDED(rv)) { + mPrefetches.AppendElement(prefetchURI); + } + } else if (confidence >= + StaticPrefs::network_predictor_preconnect_min_confidence()) { + nsCOMPtr<nsIURI> preconnectURI; + rv = NS_NewURI(getter_AddRefs(preconnectURI), uri); + if (NS_SUCCEEDED(rv)) { + mPreconnects.AppendElement(preconnectURI); + } + } else if (confidence >= + StaticPrefs::network_predictor_preresolve_min_confidence()) { + nsCOMPtr<nsIURI> preresolveURI; + rv = NS_NewURI(getter_AddRefs(preresolveURI), uri); + if (NS_SUCCEEDED(rv)) { + mPreresolves.AppendElement(preresolveURI); + } + } + + if (NS_FAILED(rv)) { + PREDICTOR_LOG( + (" NS_NewURI returned 0x%" PRIx32, static_cast<uint32_t>(rv))); + } +} + +nsresult Predictor::Prefetch(nsIURI* uri, nsIURI* referrer, + const OriginAttributes& originAttributes, + nsINetworkPredictorVerifier* verifier) { + nsAutoCString strUri, strReferrer; + uri->GetAsciiSpec(strUri); + referrer->GetAsciiSpec(strReferrer); + PREDICTOR_LOG(("Predictor::Prefetch uri=%s referrer=%s verifier=%p", + strUri.get(), strReferrer.get(), verifier)); + nsCOMPtr<nsIChannel> channel; + nsresult rv = NS_NewChannel( + getter_AddRefs(channel), uri, nsContentUtils::GetSystemPrincipal(), + nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL, + nsIContentPolicy::TYPE_OTHER, nullptr, /* nsICookieJarSettings */ + nullptr, /* aPerformanceStorage */ + nullptr, /* aLoadGroup */ + nullptr, /* aCallbacks */ + nsIRequest::LOAD_BACKGROUND); + + if (NS_FAILED(rv)) { + PREDICTOR_LOG( + (" NS_NewChannel failed rv=0x%" PRIX32, static_cast<uint32_t>(rv))); + return rv; + } + + nsCOMPtr<nsILoadInfo> loadInfo = channel->LoadInfo(); + rv = loadInfo->SetOriginAttributes(originAttributes); + + if (NS_FAILED(rv)) { + PREDICTOR_LOG( + (" Set originAttributes into loadInfo failed rv=0x%" PRIX32, + static_cast<uint32_t>(rv))); + return rv; + } + + nsCOMPtr<nsIHttpChannel> httpChannel; + httpChannel = do_QueryInterface(channel); + if (!httpChannel) { + PREDICTOR_LOG((" Could not get HTTP Channel from new channel!")); + return NS_ERROR_UNEXPECTED; + } + + nsCOMPtr<nsIReferrerInfo> referrerInfo = new dom::ReferrerInfo(referrer); + rv = httpChannel->SetReferrerInfoWithoutClone(referrerInfo); + NS_ENSURE_SUCCESS(rv, rv); + // XXX - set a header here to indicate this is a prefetch? + + nsCOMPtr<nsIStreamListener> listener = + new PrefetchListener(verifier, uri, this); + PREDICTOR_LOG((" calling AsyncOpen listener=%p channel=%p", listener.get(), + channel.get())); + rv = channel->AsyncOpen(listener); + if (NS_FAILED(rv)) { + PREDICTOR_LOG( + (" AsyncOpen failed rv=0x%" PRIX32, static_cast<uint32_t>(rv))); + } + + return rv; +} + +// Runs predictions that have been set up. +bool Predictor::RunPredictions(nsIURI* referrer, + const OriginAttributes& originAttributes, + nsINetworkPredictorVerifier* verifier) { + MOZ_ASSERT(NS_IsMainThread(), "Running prediction off main thread"); + + PREDICTOR_LOG(("Predictor::RunPredictions")); + + bool predicted = false; + uint32_t len, i; + + nsTArray<nsCOMPtr<nsIURI>> prefetches = std::move(mPrefetches), + preconnects = std::move(mPreconnects), + preresolves = std::move(mPreresolves); + + uint32_t totalPredictions = 0; + uint32_t totalPrefetches = 0; + uint32_t totalPreconnects = 0; + uint32_t totalPreresolves = 0; + + len = prefetches.Length(); + for (i = 0; i < len; ++i) { + PREDICTOR_LOG((" doing prefetch")); + nsCOMPtr<nsIURI> uri = prefetches[i]; + if (NS_SUCCEEDED(Prefetch(uri, referrer, originAttributes, verifier))) { + ++totalPredictions; + ++totalPrefetches; + predicted = true; + } + } + + len = preconnects.Length(); + for (i = 0; i < len; ++i) { + PREDICTOR_LOG((" doing preconnect")); + nsCOMPtr<nsIURI> uri = preconnects[i]; + ++totalPredictions; + ++totalPreconnects; + nsCOMPtr<nsIPrincipal> principal = + BasePrincipal::CreateContentPrincipal(uri, originAttributes); + mSpeculativeService->SpeculativeConnect(uri, principal, this, false); + predicted = true; + if (verifier) { + PREDICTOR_LOG((" sending preconnect verification")); + verifier->OnPredictPreconnect(uri); + } + } + + len = preresolves.Length(); + for (i = 0; i < len; ++i) { + nsCOMPtr<nsIURI> uri = preresolves[i]; + ++totalPredictions; + ++totalPreresolves; + nsAutoCString hostname; + uri->GetAsciiHost(hostname); + PREDICTOR_LOG((" doing preresolve %s", hostname.get())); + nsCOMPtr<nsICancelable> tmpCancelable; + mDnsService->AsyncResolveNative( + hostname, nsIDNSService::RESOLVE_TYPE_DEFAULT, + (nsIDNSService::RESOLVE_PRIORITY_MEDIUM | + nsIDNSService::RESOLVE_SPECULATE), + nullptr, mDNSListener, nullptr, originAttributes, + getter_AddRefs(tmpCancelable)); + + // Fetch HTTPS RR if needed. + if (StaticPrefs::network_dns_upgrade_with_https_rr() || + StaticPrefs::network_dns_use_https_rr_as_altsvc()) { + mDnsService->AsyncResolveNative( + hostname, nsIDNSService::RESOLVE_TYPE_HTTPSSVC, + (nsIDNSService::RESOLVE_PRIORITY_MEDIUM | + nsIDNSService::RESOLVE_SPECULATE), + nullptr, mDNSListener, nullptr, originAttributes, + getter_AddRefs(tmpCancelable)); + } + + predicted = true; + if (verifier) { + PREDICTOR_LOG((" sending preresolve verification")); + verifier->OnPredictDNS(uri); + } + } + + glean::predictor::total_predictions.AccumulateSingleSample(totalPredictions); + glean::predictor::total_prefetches.AccumulateSingleSample(totalPrefetches); + glean::predictor::total_preconnects.AccumulateSingleSample(totalPreconnects); + glean::predictor::total_preresolves.AccumulateSingleSample(totalPreresolves); + + return predicted; +} + +// Find out if a top-level page is likely to redirect. +bool Predictor::WouldRedirect(nsICacheEntry* entry, uint32_t loadCount, + uint32_t lastLoad, int32_t globalDegradation, + nsIURI** redirectURI) { + // TODO - not doing redirects for first go around + MOZ_ASSERT(NS_IsMainThread()); + + return false; +} + +NS_IMETHODIMP +Predictor::Learn(nsIURI* targetURI, nsIURI* sourceURI, + PredictorLearnReason reason, + JS::Handle<JS::Value> originAttributes, JSContext* aCx) { + OriginAttributes attrs; + + if (!originAttributes.isObject() || !attrs.Init(aCx, originAttributes)) { + return NS_ERROR_INVALID_ARG; + } + + return LearnNative(targetURI, sourceURI, reason, attrs); +} + +// Called from the main thread to update the database +NS_IMETHODIMP +Predictor::LearnNative(nsIURI* targetURI, nsIURI* sourceURI, + PredictorLearnReason reason, + const OriginAttributes& originAttributes) { + MOZ_ASSERT(NS_IsMainThread(), + "Predictor interface methods must be called on the main thread"); + + PREDICTOR_LOG(("Predictor::Learn")); + + if (!StaticPrefs::network_predictor_enabled()) { + PREDICTOR_LOG((" not enabled")); + return NS_OK; + } + + if (IsNeckoChild()) { + if (!gNeckoChild) { + return NS_ERROR_FAILURE; + } + + PREDICTOR_LOG((" called on child process")); + + RefPtr<PredictorLearnRunnable> runnable = new PredictorLearnRunnable( + targetURI, sourceURI, reason, originAttributes); + SchedulerGroup::Dispatch(runnable.forget()); + return NS_OK; + } + + PREDICTOR_LOG((" called on parent process")); + + if (!mInitialized) { + PREDICTOR_LOG((" not initialized")); + return NS_OK; + } + + if (originAttributes.IsPrivateBrowsing()) { + // Don't want to do anything in PB mode + PREDICTOR_LOG((" in PB mode")); + return NS_OK; + } + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + PREDICTOR_LOG((" got non-HTTP[S] URI")); + return NS_ERROR_INVALID_ARG; + } + + nsCOMPtr<nsIURI> targetOrigin; + nsCOMPtr<nsIURI> sourceOrigin; + nsCOMPtr<nsIURI> uriKey; + nsCOMPtr<nsIURI> originKey; + nsresult rv; + + switch (reason) { + case nsINetworkPredictor::LEARN_LOAD_TOPLEVEL: + if (!targetURI || sourceURI) { + PREDICTOR_LOG((" load toplevel invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + rv = ExtractOrigin(targetURI, getter_AddRefs(targetOrigin)); + NS_ENSURE_SUCCESS(rv, rv); + uriKey = targetURI; + originKey = targetOrigin; + break; + case nsINetworkPredictor::LEARN_STARTUP: + if (!targetURI || sourceURI) { + PREDICTOR_LOG((" startup invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + rv = ExtractOrigin(targetURI, getter_AddRefs(targetOrigin)); + NS_ENSURE_SUCCESS(rv, rv); + uriKey = mStartupURI; + originKey = mStartupURI; + break; + case nsINetworkPredictor::LEARN_LOAD_REDIRECT: + case nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE: + if (!targetURI || !sourceURI) { + PREDICTOR_LOG((" redirect/subresource invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + rv = ExtractOrigin(targetURI, getter_AddRefs(targetOrigin)); + NS_ENSURE_SUCCESS(rv, rv); + rv = ExtractOrigin(sourceURI, getter_AddRefs(sourceOrigin)); + NS_ENSURE_SUCCESS(rv, rv); + uriKey = sourceURI; + originKey = sourceOrigin; + break; + default: + PREDICTOR_LOG((" invalid reason")); + return NS_ERROR_INVALID_ARG; + } + + uint32_t learnAttempts = 1; + Predictor::Reason argReason{}; + argReason.mLearn = reason; + + // We always open the full uri (general cache) entry first, so we don't gum up + // the works waiting on predictor-only entries to open + RefPtr<Predictor::Action> uriAction = new Predictor::Action( + Predictor::Action::IS_FULL_URI, Predictor::Action::DO_LEARN, argReason, + targetURI, sourceURI, nullptr, this); + nsAutoCString uriKeyStr, targetUriStr, sourceUriStr; + uriKey->GetAsciiSpec(uriKeyStr); + targetURI->GetAsciiSpec(targetUriStr); + if (sourceURI) { + sourceURI->GetAsciiSpec(sourceUriStr); + } + PREDICTOR_LOG( + (" Learn uriKey=%s targetURI=%s sourceURI=%s reason=%d " + "action=%p", + uriKeyStr.get(), targetUriStr.get(), sourceUriStr.get(), reason, + uriAction.get())); + + nsCOMPtr<nsICacheStorage> cacheDiskStorage; + + RefPtr<LoadContextInfo> lci = new LoadContextInfo(false, originAttributes); + + rv = mCacheStorageService->DiskCacheStorage(lci, + getter_AddRefs(cacheDiskStorage)); + NS_ENSURE_SUCCESS(rv, rv); + + // For learning full URI things, we *always* open readonly and secretly, as we + // rely on actual pageloads to update the entry's metadata for us. + uint32_t uriOpenFlags = nsICacheStorage::OPEN_READONLY | + nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::CHECK_MULTITHREADED; + if (reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL) { + // Learning for toplevel we want to open the full uri entry priority, since + // it's likely this entry will be used soon anyway, and we want this to be + // opened ASAP. + uriOpenFlags |= nsICacheStorage::OPEN_PRIORITY; + } + cacheDiskStorage->AsyncOpenURI(uriKey, ""_ns, uriOpenFlags, uriAction); + + // Now we open the origin-only (and therefore predictor-only) entry + RefPtr<Predictor::Action> originAction = new Predictor::Action( + Predictor::Action::IS_ORIGIN, Predictor::Action::DO_LEARN, argReason, + targetOrigin, sourceOrigin, nullptr, this); + nsAutoCString originKeyStr, targetOriginStr, sourceOriginStr; + originKey->GetAsciiSpec(originKeyStr); + targetOrigin->GetAsciiSpec(targetOriginStr); + if (sourceOrigin) { + sourceOrigin->GetAsciiSpec(sourceOriginStr); + } + PREDICTOR_LOG( + (" Learn originKey=%s targetOrigin=%s sourceOrigin=%s reason=%d " + "action=%p", + originKeyStr.get(), targetOriginStr.get(), sourceOriginStr.get(), reason, + originAction.get())); + uint32_t originOpenFlags; + if (reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL) { + // This is the only case when we want to update the 'last used' metadata on + // the cache entry we're getting. This only applies to predictor-specific + // entries. + originOpenFlags = + nsICacheStorage::OPEN_NORMALLY | nsICacheStorage::CHECK_MULTITHREADED; + } else { + originOpenFlags = nsICacheStorage::OPEN_READONLY | + nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::CHECK_MULTITHREADED; + } + cacheDiskStorage->AsyncOpenURI(originKey, + nsLiteralCString(PREDICTOR_ORIGIN_EXTENSION), + originOpenFlags, originAction); + + glean::predictor::learn_attempts.AccumulateSingleSample(learnAttempts); + PREDICTOR_LOG(("Predictor::Learn returning")); + return NS_OK; +} + +void Predictor::LearnInternal(PredictorLearnReason reason, nsICacheEntry* entry, + bool isNew, bool fullUri, nsIURI* targetURI, + nsIURI* sourceURI) { + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::LearnInternal")); + + nsCString junk; + if (!fullUri && reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL && + NS_FAILED( + entry->GetMetaDataElement(SEEN_META_DATA, getter_Copies(junk)))) { + // This is an origin-only entry that we haven't seen before. Let's mark it + // as seen. + PREDICTOR_LOG((" marking new origin entry as seen")); + nsresult rv = entry->SetMetaDataElement(SEEN_META_DATA, "1"); + if (NS_FAILED(rv)) { + PREDICTOR_LOG((" failed to mark origin entry seen")); + return; + } + + // Need to ensure someone else can get to the entry if necessary + entry->MetaDataReady(); + return; + } + + switch (reason) { + case nsINetworkPredictor::LEARN_LOAD_TOPLEVEL: + // This case only exists to be used during tests - code outside the + // predictor tests should NEVER call Learn with LEARN_LOAD_TOPLEVEL. + // The predictor xpcshell test needs this branch, however, because we + // have no real page loads in xpcshell, and this is how we fake it up + // so that all the work that normally happens behind the scenes in a + // page load can be done for testing purposes. + if (fullUri && StaticPrefs::network_predictor_doing_tests()) { + PREDICTOR_LOG( + (" WARNING - updating rolling load count. " + "If you see this outside tests, you did it wrong")); + + // Since the visitor gets called under a cache lock, all we do there is + // get copies of the keys/values we care about, and then do the real + // work here + entry->VisitMetaData(this); + nsTArray<nsCString> keysToOperateOn = std::move(mKeysToOperateOn), + valuesToOperateOn = std::move(mValuesToOperateOn); + + MOZ_ASSERT(keysToOperateOn.Length() == valuesToOperateOn.Length()); + for (size_t i = 0; i < keysToOperateOn.Length(); ++i) { + const char* key = keysToOperateOn[i].BeginReading(); + const char* value = valuesToOperateOn[i].BeginReading(); + + nsCString uri; + uint32_t hitCount, lastHit, flags; + if (!ParseMetaDataEntry(key, value, uri, hitCount, lastHit, flags)) { + // This failed, get rid of it so we don't waste space + entry->SetMetaDataElement(key, nullptr); + continue; + } + UpdateRollingLoadCount(entry, flags, key, hitCount, lastHit); + } + } else { + PREDICTOR_LOG((" nothing to do for toplevel")); + } + break; + case nsINetworkPredictor::LEARN_LOAD_REDIRECT: + if (fullUri) { + LearnForRedirect(entry, targetURI); + } + break; + case nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE: + LearnForSubresource(entry, targetURI); + break; + case nsINetworkPredictor::LEARN_STARTUP: + LearnForStartup(entry, targetURI); + break; + default: + PREDICTOR_LOG((" unexpected reason value")); + MOZ_ASSERT(false, "Got unexpected value for learn reason!"); + } +} + +NS_IMPL_ISUPPORTS(Predictor::SpaceCleaner, nsICacheEntryMetaDataVisitor) + +NS_IMETHODIMP +Predictor::SpaceCleaner::OnMetaDataElement(const char* key, const char* value) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsURIMetadataElement(key)) { + // This isn't a bit of metadata we care about + return NS_OK; + } + + nsCString uri; + uint32_t hitCount, lastHit, flags; + bool ok = + mPredictor->ParseMetaDataEntry(key, value, uri, hitCount, lastHit, flags); + + if (!ok) { + // Couldn't parse this one, just get rid of it + nsCString nsKey; + nsKey.AssignASCII(key); + mLongKeysToDelete.AppendElement(nsKey); + return NS_OK; + } + + uint32_t uriLength = uri.Length(); + if (uriLength > StaticPrefs::network_predictor_max_uri_length()) { + // Default to getting rid of URIs that are too long and were put in before + // we had our limit on URI length, in order to free up some space. + nsCString nsKey; + nsKey.AssignASCII(key); + mLongKeysToDelete.AppendElement(nsKey); + return NS_OK; + } + + if (!mLRUKeyToDelete || lastHit < mLRUStamp) { + mLRUKeyToDelete = key; + mLRUStamp = lastHit; + } + + return NS_OK; +} + +void Predictor::SpaceCleaner::Finalize(nsICacheEntry* entry) { + MOZ_ASSERT(NS_IsMainThread()); + + if (mLRUKeyToDelete) { + entry->SetMetaDataElement(mLRUKeyToDelete, nullptr); + } + + for (size_t i = 0; i < mLongKeysToDelete.Length(); ++i) { + entry->SetMetaDataElement(mLongKeysToDelete[i].BeginReading(), nullptr); + } +} + +// Called when a subresource has been hit from a top-level load. Uses the two +// helper functions above to update the database appropriately. +void Predictor::LearnForSubresource(nsICacheEntry* entry, nsIURI* targetURI) { + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::LearnForSubresource")); + + uint32_t lastLoad; + nsresult rv = entry->GetLastFetched(&lastLoad); + NS_ENSURE_SUCCESS_VOID(rv); + + uint32_t loadCount; + rv = entry->GetFetchCount(&loadCount); + NS_ENSURE_SUCCESS_VOID(rv); + + nsCString key; + key.AssignLiteral(META_DATA_PREFIX); + nsCString uri; + targetURI->GetAsciiSpec(uri); + key.Append(uri); + if (uri.Length() > StaticPrefs::network_predictor_max_uri_length()) { + // We do this to conserve space/prevent OOMs + PREDICTOR_LOG((" uri too long!")); + entry->SetMetaDataElement(key.BeginReading(), nullptr); + return; + } + + nsCString value; + rv = entry->GetMetaDataElement(key.BeginReading(), getter_Copies(value)); + + uint32_t hitCount, lastHit, flags; + bool isNewResource = + (NS_FAILED(rv) || + !ParseMetaDataEntry(key.BeginReading(), value.BeginReading(), uri, + hitCount, lastHit, flags)); + + int32_t resourceCount = 0; + if (isNewResource) { + // This is a new addition + PREDICTOR_LOG((" new resource")); + nsCString s; + rv = entry->GetMetaDataElement(RESOURCE_META_DATA, getter_Copies(s)); + if (NS_SUCCEEDED(rv)) { + resourceCount = atoi(s.BeginReading()); + } + if (resourceCount >= + StaticPrefs::network_predictor_max_resources_per_entry()) { + RefPtr<Predictor::SpaceCleaner> cleaner = + new Predictor::SpaceCleaner(this); + entry->VisitMetaData(cleaner); + cleaner->Finalize(entry); + } else { + ++resourceCount; + } + nsAutoCString count; + count.AppendInt(resourceCount); + rv = entry->SetMetaDataElement(RESOURCE_META_DATA, count.BeginReading()); + if (NS_FAILED(rv)) { + PREDICTOR_LOG((" failed to update resource count")); + return; + } + hitCount = 1; + flags = 0; + } else { + PREDICTOR_LOG((" existing resource")); + hitCount = std::min(hitCount + 1, loadCount); + } + + // Update the rolling load count to mark this sub-resource as seen on the + // most-recent pageload so it can be eligible for prefetch (assuming all + // the other stars align). + flags |= (1 << kRollingLoadOffset); + + nsCString newValue; + MakeMetadataEntry(hitCount, lastLoad, flags, newValue); + rv = entry->SetMetaDataElement(key.BeginReading(), newValue.BeginReading()); + PREDICTOR_LOG( + (" SetMetaDataElement -> 0x%08" PRIX32, static_cast<uint32_t>(rv))); + if (NS_FAILED(rv) && isNewResource) { + // Roll back the increment to the resource count we made above. + PREDICTOR_LOG((" rolling back resource count update")); + --resourceCount; + if (resourceCount == 0) { + entry->SetMetaDataElement(RESOURCE_META_DATA, nullptr); + } else { + nsAutoCString count; + count.AppendInt(resourceCount); + entry->SetMetaDataElement(RESOURCE_META_DATA, count.BeginReading()); + } + } +} + +// This is called when a top-level loaded ended up redirecting to a different +// URI so we can keep track of that fact. +void Predictor::LearnForRedirect(nsICacheEntry* entry, nsIURI* targetURI) { + MOZ_ASSERT(NS_IsMainThread()); + + // TODO - not doing redirects for first go around + PREDICTOR_LOG(("Predictor::LearnForRedirect")); +} + +// This will add a page to our list of startup pages if it's being loaded +// before our startup window has expired. +void Predictor::MaybeLearnForStartup(nsIURI* uri, bool fullUri, + const OriginAttributes& originAttributes) { + MOZ_ASSERT(NS_IsMainThread()); + + // TODO - not doing startup for first go around + PREDICTOR_LOG(("Predictor::MaybeLearnForStartup")); +} + +// Add information about a top-level load to our list of startup pages +void Predictor::LearnForStartup(nsICacheEntry* entry, nsIURI* targetURI) { + MOZ_ASSERT(NS_IsMainThread()); + + // These actually do the same set of work, just on different entries, so we + // can pass through to get the real work done here + PREDICTOR_LOG(("Predictor::LearnForStartup")); + LearnForSubresource(entry, targetURI); +} + +bool Predictor::ParseMetaDataEntry(const char* key, const char* value, + nsCString& uri, uint32_t& hitCount, + uint32_t& lastHit, uint32_t& flags) { + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG( + ("Predictor::ParseMetaDataEntry key=%s value=%s", key ? key : "", value)); + + const char* comma = strchr(value, ','); + if (!comma) { + PREDICTOR_LOG((" could not find first comma")); + return false; + } + + uint32_t version = static_cast<uint32_t>(atoi(value)); + PREDICTOR_LOG((" version -> %u", version)); + + if (version != METADATA_VERSION) { + PREDICTOR_LOG( + (" metadata version mismatch %u != %u", version, METADATA_VERSION)); + return false; + } + + value = comma + 1; + comma = strchr(value, ','); + if (!comma) { + PREDICTOR_LOG((" could not find second comma")); + return false; + } + + hitCount = static_cast<uint32_t>(atoi(value)); + PREDICTOR_LOG((" hitCount -> %u", hitCount)); + + value = comma + 1; + comma = strchr(value, ','); + if (!comma) { + PREDICTOR_LOG((" could not find third comma")); + return false; + } + + lastHit = static_cast<uint32_t>(atoi(value)); + PREDICTOR_LOG((" lastHit -> %u", lastHit)); + + value = comma + 1; + flags = static_cast<uint32_t>(atoi(value)); + PREDICTOR_LOG((" flags -> %u", flags)); + + if (key) { + const char* uriStart = key + (sizeof(META_DATA_PREFIX) - 1); + uri.AssignASCII(uriStart); + PREDICTOR_LOG((" uri -> %s", uriStart)); + } else { + uri.Truncate(); + } + + return true; +} + +NS_IMETHODIMP +Predictor::Reset() { + MOZ_ASSERT(NS_IsMainThread(), + "Predictor interface methods must be called on the main thread"); + + PREDICTOR_LOG(("Predictor::Reset")); + + if (!StaticPrefs::network_predictor_enabled()) { + PREDICTOR_LOG((" not enabled")); + return NS_OK; + } + + if (IsNeckoChild()) { + if (!gNeckoChild) { + return NS_ERROR_FAILURE; + } + + PREDICTOR_LOG((" forwarding to parent process")); + gNeckoChild->SendPredReset(); + return NS_OK; + } + + PREDICTOR_LOG((" called on parent process")); + + if (!mInitialized) { + PREDICTOR_LOG((" not initialized")); + return NS_OK; + } + + RefPtr<Predictor::Resetter> reset = new Predictor::Resetter(this); + PREDICTOR_LOG((" created a resetter")); + mCacheStorageService->AsyncVisitAllStorages(reset, true); + PREDICTOR_LOG((" Cache async launched, returning now")); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(Predictor::Resetter, nsICacheEntryOpenCallback, + nsICacheEntryMetaDataVisitor, nsICacheStorageVisitor); + +Predictor::Resetter::Resetter(Predictor* predictor) + : mEntriesToVisit(0), mPredictor(predictor) {} + +NS_IMETHODIMP +Predictor::Resetter::OnCacheEntryCheck(nsICacheEntry* entry, uint32_t* result) { + *result = nsICacheEntryOpenCallback::ENTRY_WANTED; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnCacheEntryAvailable(nsICacheEntry* entry, bool isNew, + nsresult result) { + MOZ_ASSERT(NS_IsMainThread()); + + if (NS_FAILED(result)) { + // This can happen when we've tried to open an entry that doesn't exist for + // some non-reset operation, and then get reset shortly thereafter (as + // happens in some of our tests). + --mEntriesToVisit; + if (!mEntriesToVisit) { + Complete(); + } + return NS_OK; + } + + entry->VisitMetaData(this); + nsTArray<nsCString> keysToDelete = std::move(mKeysToDelete); + + for (size_t i = 0; i < keysToDelete.Length(); ++i) { + const char* key = keysToDelete[i].BeginReading(); + entry->SetMetaDataElement(key, nullptr); + } + + --mEntriesToVisit; + if (!mEntriesToVisit) { + Complete(); + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnMetaDataElement(const char* asciiKey, + const char* asciiValue) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!StringBeginsWith(nsDependentCString(asciiKey), + nsLiteralCString(META_DATA_PREFIX))) { + // Not a metadata entry we care about, carry on + return NS_OK; + } + + nsCString key; + key.AssignASCII(asciiKey); + mKeysToDelete.AppendElement(key); + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnCacheStorageInfo(uint32_t entryCount, + uint64_t consumption, uint64_t capacity, + nsIFile* diskDirectory) { + MOZ_ASSERT(NS_IsMainThread()); + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnCacheEntryInfo(nsIURI* uri, const nsACString& idEnhance, + int64_t dataSize, int64_t altDataSize, + uint32_t fetchCount, + uint32_t lastModifiedTime, + uint32_t expirationTime, bool aPinned, + nsILoadContextInfo* aInfo) { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv; + + // The predictor will only ever touch entries with no idEnhance ("") or an + // idEnhance of PREDICTOR_ORIGIN_EXTENSION, so we filter out any entries that + // don't match that to avoid doing extra work. + if (idEnhance.EqualsLiteral(PREDICTOR_ORIGIN_EXTENSION)) { + // This is an entry we own, so we can just doom it entirely + nsCOMPtr<nsICacheStorage> cacheDiskStorage; + + rv = mPredictor->mCacheStorageService->DiskCacheStorage( + aInfo, getter_AddRefs(cacheDiskStorage)); + + NS_ENSURE_SUCCESS(rv, rv); + cacheDiskStorage->AsyncDoomURI(uri, idEnhance, nullptr); + } else if (idEnhance.IsEmpty()) { + // This is an entry we don't own, so we have to be a little more careful and + // just get rid of our own metadata entries. Append it to an array of things + // to operate on and then do the operations later so we don't end up calling + // Complete() multiple times/too soon. + ++mEntriesToVisit; + mURIsToVisit.AppendElement(uri); + mInfosToVisit.AppendElement(aInfo); + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnCacheEntryVisitCompleted() { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv; + + nsTArray<nsCOMPtr<nsIURI>> urisToVisit = std::move(mURIsToVisit); + + MOZ_ASSERT(mEntriesToVisit == urisToVisit.Length()); + + nsTArray<nsCOMPtr<nsILoadContextInfo>> infosToVisit = + std::move(mInfosToVisit); + + MOZ_ASSERT(mEntriesToVisit == infosToVisit.Length()); + + if (!mEntriesToVisit) { + Complete(); + return NS_OK; + } + + uint32_t entriesToVisit = urisToVisit.Length(); + for (uint32_t i = 0; i < entriesToVisit; ++i) { + nsCString u; + nsCOMPtr<nsICacheStorage> cacheDiskStorage; + + rv = mPredictor->mCacheStorageService->DiskCacheStorage( + infosToVisit[i], getter_AddRefs(cacheDiskStorage)); + NS_ENSURE_SUCCESS(rv, rv); + + urisToVisit[i]->GetAsciiSpec(u); + rv = cacheDiskStorage->AsyncOpenURI( + urisToVisit[i], ""_ns, + nsICacheStorage::OPEN_READONLY | nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::CHECK_MULTITHREADED, + this); + if (NS_FAILED(rv)) { + mEntriesToVisit--; + if (!mEntriesToVisit) { + Complete(); + return NS_OK; + } + } + } + + return NS_OK; +} + +void Predictor::Resetter::Complete() { + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService(); + if (!obs) { + PREDICTOR_LOG(("COULD NOT GET OBSERVER SERVICE!")); + return; + } + + obs->NotifyObservers(nullptr, "predictor-reset-complete", nullptr); +} + +// Helper functions to make using the predictor easier from native code + +static StaticRefPtr<nsINetworkPredictor> sPredictor; + +static nsresult EnsureGlobalPredictor(nsINetworkPredictor** aPredictor) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!sPredictor) { + nsresult rv; + nsCOMPtr<nsINetworkPredictor> predictor; + predictor = mozilla::components::Predictor::Service(&rv); + NS_ENSURE_SUCCESS(rv, rv); + sPredictor = predictor; + ClearOnShutdown(&sPredictor); + } + + nsCOMPtr<nsINetworkPredictor> predictor = sPredictor.get(); + predictor.forget(aPredictor); + return NS_OK; +} + +nsresult PredictorPredict(nsIURI* targetURI, nsIURI* sourceURI, + PredictorPredictReason reason, + const OriginAttributes& originAttributes, + nsINetworkPredictorVerifier* verifier) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr<nsINetworkPredictor> predictor; + nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + return predictor->PredictNative(targetURI, sourceURI, reason, + originAttributes, verifier); +} + +nsresult PredictorLearn(nsIURI* targetURI, nsIURI* sourceURI, + PredictorLearnReason reason, + const OriginAttributes& originAttributes) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr<nsINetworkPredictor> predictor; + nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + return predictor->LearnNative(targetURI, sourceURI, reason, originAttributes); +} + +nsresult PredictorLearn(nsIURI* targetURI, nsIURI* sourceURI, + PredictorLearnReason reason, nsILoadGroup* loadGroup) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr<nsINetworkPredictor> predictor; + nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsILoadContext> loadContext; + OriginAttributes originAttributes; + + if (loadGroup) { + nsCOMPtr<nsIInterfaceRequestor> callbacks; + loadGroup->GetNotificationCallbacks(getter_AddRefs(callbacks)); + if (callbacks) { + loadContext = do_GetInterface(callbacks); + + if (loadContext) { + loadContext->GetOriginAttributes(originAttributes); + } + } + } + + return predictor->LearnNative(targetURI, sourceURI, reason, originAttributes); +} + +nsresult PredictorLearn(nsIURI* targetURI, nsIURI* sourceURI, + PredictorLearnReason reason, dom::Document* document) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr<nsINetworkPredictor> predictor; + nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + OriginAttributes originAttributes; + + if (document) { + nsCOMPtr<nsIPrincipal> docPrincipal = document->NodePrincipal(); + + if (docPrincipal) { + originAttributes = docPrincipal->OriginAttributesRef(); + } + } + + return predictor->LearnNative(targetURI, sourceURI, reason, originAttributes); +} + +nsresult PredictorLearnRedirect(nsIURI* targetURI, nsIChannel* channel, + const OriginAttributes& originAttributes) { + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr<nsIURI> sourceURI; + nsresult rv = channel->GetOriginalURI(getter_AddRefs(sourceURI)); + NS_ENSURE_SUCCESS(rv, rv); + + bool sameUri; + rv = targetURI->Equals(sourceURI, &sameUri); + NS_ENSURE_SUCCESS(rv, rv); + + if (sameUri) { + return NS_OK; + } + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr<nsINetworkPredictor> predictor; + rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + return predictor->LearnNative(targetURI, sourceURI, + nsINetworkPredictor::LEARN_LOAD_REDIRECT, + originAttributes); +} + +// nsINetworkPredictorVerifier + +/** + * Call through to the child's verifier (only during tests) + */ +NS_IMETHODIMP +Predictor::OnPredictPrefetch(nsIURI* aURI, uint32_t httpStatus) { + if (IsNeckoChild()) { + if (mChildVerifier) { + // Ideally, we'd assert here. But since we're slowly moving towards a + // world where we have multiple child processes, and only one child + // process will be likely to have a verifier, we have to play it safer. + return mChildVerifier->OnPredictPrefetch(aURI, httpStatus); + } + return NS_OK; + } + + MOZ_DIAGNOSTIC_ASSERT(aURI, "aURI must not be null"); + + for (auto* cp : dom::ContentParent::AllProcesses(dom::ContentParent::eLive)) { + PNeckoParent* neckoParent = SingleManagedOrNull(cp->ManagedPNeckoParent()); + if (!neckoParent) { + continue; + } + if (!neckoParent->SendPredOnPredictPrefetch(aURI, httpStatus)) { + return NS_ERROR_NOT_AVAILABLE; + } + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::OnPredictPreconnect(nsIURI* aURI) { + if (IsNeckoChild()) { + if (mChildVerifier) { + // Ideally, we'd assert here. But since we're slowly moving towards a + // world where we have multiple child processes, and only one child + // process will be likely to have a verifier, we have to play it safer. + return mChildVerifier->OnPredictPreconnect(aURI); + } + return NS_OK; + } + + MOZ_DIAGNOSTIC_ASSERT(aURI, "aURI must not be null"); + + for (auto* cp : dom::ContentParent::AllProcesses(dom::ContentParent::eLive)) { + PNeckoParent* neckoParent = SingleManagedOrNull(cp->ManagedPNeckoParent()); + if (!neckoParent) { + continue; + } + if (!neckoParent->SendPredOnPredictPreconnect(aURI)) { + return NS_ERROR_NOT_AVAILABLE; + } + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::OnPredictDNS(nsIURI* aURI) { + if (IsNeckoChild()) { + if (mChildVerifier) { + // Ideally, we'd assert here. But since we're slowly moving towards a + // world where we have multiple child processes, and only one child + // process will be likely to have a verifier, we have to play it safer. + return mChildVerifier->OnPredictDNS(aURI); + } + return NS_OK; + } + + MOZ_DIAGNOSTIC_ASSERT(aURI, "aURI must not be null"); + + for (auto* cp : dom::ContentParent::AllProcesses(dom::ContentParent::eLive)) { + PNeckoParent* neckoParent = SingleManagedOrNull(cp->ManagedPNeckoParent()); + if (!neckoParent) { + continue; + } + if (!neckoParent->SendPredOnPredictDNS(aURI)) { + return NS_ERROR_NOT_AVAILABLE; + } + } + + return NS_OK; +} + +// Predictor::PrefetchListener +// nsISupports +NS_IMPL_ISUPPORTS(Predictor::PrefetchListener, nsIStreamListener, + nsIRequestObserver) + +// nsIRequestObserver +NS_IMETHODIMP +Predictor::PrefetchListener::OnStartRequest(nsIRequest* aRequest) { + mStartTime = TimeStamp::Now(); + return NS_OK; +} + +NS_IMETHODIMP +Predictor::PrefetchListener::OnStopRequest(nsIRequest* aRequest, + nsresult aStatusCode) { + PREDICTOR_LOG(("OnStopRequest this=%p aStatusCode=0x%" PRIX32, this, + static_cast<uint32_t>(aStatusCode))); + NS_ENSURE_ARG(aRequest); + if (NS_FAILED(aStatusCode)) { + return aStatusCode; + } + glean::predictor::prefetch_time.AccumulateRawDuration(TimeStamp::Now() - + mStartTime); + + nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aRequest); + if (!httpChannel) { + PREDICTOR_LOG((" Could not get HTTP Channel!")); + return NS_ERROR_UNEXPECTED; + } + nsCOMPtr<nsICachingChannel> cachingChannel = do_QueryInterface(httpChannel); + if (!cachingChannel) { + PREDICTOR_LOG((" Could not get caching channel!")); + return NS_ERROR_UNEXPECTED; + } + + nsresult rv = NS_OK; + uint32_t httpStatus; + rv = httpChannel->GetResponseStatus(&httpStatus); + if (NS_SUCCEEDED(rv) && httpStatus == 200) { + rv = cachingChannel->ForceCacheEntryValidFor( + StaticPrefs::network_predictor_prefetch_force_valid_for()); + PREDICTOR_LOG((" forcing entry valid for %d seconds rv=%" PRIX32, + StaticPrefs::network_predictor_prefetch_force_valid_for(), + static_cast<uint32_t>(rv))); + } else { + rv = cachingChannel->ForceCacheEntryValidFor(0); + glean::predictor::prefetch_use_status + .EnumGet(glean::predictor::PrefetchUseStatusLabel::eNot200) + .Add(); + PREDICTOR_LOG((" removing any forced validity rv=%" PRIX32, + static_cast<uint32_t>(rv))); + } + + nsAutoCString reqName; + rv = aRequest->GetName(reqName); + if (NS_FAILED(rv)) { + reqName.AssignLiteral("<unknown>"); + } + + PREDICTOR_LOG((" request %s status %u", reqName.get(), httpStatus)); + + if (mVerifier) { + mVerifier->OnPredictPrefetch(mURI, httpStatus); + } + + return rv; +} + +// nsIStreamListener +NS_IMETHODIMP +Predictor::PrefetchListener::OnDataAvailable(nsIRequest* aRequest, + nsIInputStream* aInputStream, + uint64_t aOffset, + const uint32_t aCount) { + uint32_t result; + return aInputStream->ReadSegments(NS_DiscardSegment, nullptr, aCount, + &result); +} + +// Miscellaneous Predictor + +void Predictor::UpdateCacheability(nsIURI* sourceURI, nsIURI* targetURI, + uint32_t httpStatus, + nsHttpRequestHead& requestHead, + nsHttpResponseHead* responseHead, + nsILoadContextInfo* lci, bool isTracking) { + MOZ_ASSERT(NS_IsMainThread()); + + if (lci && lci->IsPrivate()) { + PREDICTOR_LOG(("Predictor::UpdateCacheability in PB mode - ignoring")); + return; + } + + if (!sourceURI || !targetURI) { + PREDICTOR_LOG( + ("Predictor::UpdateCacheability missing source or target uri")); + return; + } + + if (!IsNullOrHttp(sourceURI) || !IsNullOrHttp(targetURI)) { + PREDICTOR_LOG(("Predictor::UpdateCacheability non-http(s) uri")); + return; + } + + RefPtr<Predictor> self = sSelf; + if (self) { + nsAutoCString method; + requestHead.Method(method); + + nsAutoCString vary; + (void)responseHead->GetHeader(nsHttp::Vary, vary); + + nsAutoCString cacheControlHeader; + (void)responseHead->GetHeader(nsHttp::Cache_Control, cacheControlHeader); + CacheControlParser cacheControl(cacheControlHeader); + + self->UpdateCacheabilityInternal(sourceURI, targetURI, httpStatus, method, + *lci->OriginAttributesPtr(), isTracking, + !vary.IsEmpty(), cacheControl.NoStore()); + } +} + +void Predictor::UpdateCacheabilityInternal( + nsIURI* sourceURI, nsIURI* targetURI, uint32_t httpStatus, + const nsCString& method, const OriginAttributes& originAttributes, + bool isTracking, bool couldVary, bool isNoStore) { + PREDICTOR_LOG(("Predictor::UpdateCacheability httpStatus=%u", httpStatus)); + + nsresult rv; + + if (!mInitialized) { + PREDICTOR_LOG((" not initialized")); + return; + } + + if (!StaticPrefs::network_predictor_enabled()) { + PREDICTOR_LOG((" not enabled")); + return; + } + + nsCOMPtr<nsICacheStorage> cacheDiskStorage; + + RefPtr<LoadContextInfo> lci = new LoadContextInfo(false, originAttributes); + + rv = mCacheStorageService->DiskCacheStorage(lci, + getter_AddRefs(cacheDiskStorage)); + if (NS_FAILED(rv)) { + PREDICTOR_LOG((" cannot get disk cache storage")); + return; + } + + uint32_t openFlags = nsICacheStorage::OPEN_READONLY | + nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::CHECK_MULTITHREADED; + RefPtr<Predictor::CacheabilityAction> action = + new Predictor::CacheabilityAction(targetURI, httpStatus, method, + isTracking, couldVary, isNoStore, this); + nsAutoCString uri; + targetURI->GetAsciiSpec(uri); + PREDICTOR_LOG((" uri=%s action=%p", uri.get(), action.get())); + cacheDiskStorage->AsyncOpenURI(sourceURI, ""_ns, openFlags, action); +} + +NS_IMPL_ISUPPORTS(Predictor::CacheabilityAction, nsICacheEntryOpenCallback, + nsICacheEntryMetaDataVisitor); + +NS_IMETHODIMP +Predictor::CacheabilityAction::OnCacheEntryCheck(nsICacheEntry* entry, + uint32_t* result) { + *result = nsICacheEntryOpenCallback::ENTRY_WANTED; + return NS_OK; +} + +namespace { +enum PrefetchDecisionReason { + PREFETCHABLE, + STATUS_NOT_200, + METHOD_NOT_GET, + URL_HAS_QUERY_STRING, + RESOURCE_IS_TRACKING, + RESOURCE_COULD_VARY, + RESOURCE_IS_NO_STORE +}; +} + +NS_IMETHODIMP +Predictor::CacheabilityAction::OnCacheEntryAvailable(nsICacheEntry* entry, + bool isNew, + nsresult result) { + MOZ_ASSERT(NS_IsMainThread()); + // This is being opened read-only, so isNew should always be false + MOZ_ASSERT(!isNew); + + PREDICTOR_LOG(("CacheabilityAction::OnCacheEntryAvailable this=%p", this)); + if (NS_FAILED(result)) { + // Nothing to do + PREDICTOR_LOG((" nothing to do result=%" PRIX32 " isNew=%d", + static_cast<uint32_t>(result), isNew)); + return NS_OK; + } + + nsCString strTargetURI; + nsresult rv = mTargetURI->GetAsciiSpec(strTargetURI); + if (NS_FAILED(rv)) { + PREDICTOR_LOG( + (" GetAsciiSpec returned %" PRIx32, static_cast<uint32_t>(rv))); + return NS_OK; + } + + rv = entry->VisitMetaData(this); + if (NS_FAILED(rv)) { + PREDICTOR_LOG( + (" VisitMetaData returned %" PRIx32, static_cast<uint32_t>(rv))); + return NS_OK; + } + + nsTArray<nsCString> keysToCheck = std::move(mKeysToCheck), + valuesToCheck = std::move(mValuesToCheck); + + bool hasQueryString = false; + nsAutoCString query; + if (NS_SUCCEEDED(mTargetURI->GetQuery(query)) && !query.IsEmpty()) { + hasQueryString = true; + } + + MOZ_ASSERT(keysToCheck.Length() == valuesToCheck.Length()); + for (size_t i = 0; i < keysToCheck.Length(); ++i) { + const char* key = keysToCheck[i].BeginReading(); + const char* value = valuesToCheck[i].BeginReading(); + nsCString uri; + uint32_t hitCount, lastHit, flags; + + if (!mPredictor->ParseMetaDataEntry(key, value, uri, hitCount, lastHit, + flags)) { + PREDICTOR_LOG((" failed to parse key=%s value=%s", key, value)); + continue; + } + + if (strTargetURI.Equals(uri)) { + bool prefetchable = true; + PrefetchDecisionReason reason = PREFETCHABLE; + + if (mHttpStatus != 200) { + prefetchable = false; + reason = STATUS_NOT_200; + } else if (!mMethod.EqualsLiteral("GET")) { + prefetchable = false; + reason = METHOD_NOT_GET; + } else if (hasQueryString) { + prefetchable = false; + reason = URL_HAS_QUERY_STRING; + } else if (mIsTracking) { + prefetchable = false; + reason = RESOURCE_IS_TRACKING; + } else if (mCouldVary) { + prefetchable = false; + reason = RESOURCE_COULD_VARY; + } else if (mIsNoStore) { + // We don't set prefetchable = false yet, because we just want to know + // what kind of effect this would have on prefetching. + reason = RESOURCE_IS_NO_STORE; + } + + glean::predictor::prefetch_decision_reason.AccumulateSingleSample(reason); + + if (prefetchable) { + PREDICTOR_LOG((" marking %s cacheable", key)); + flags |= FLAG_PREFETCHABLE; + } else { + PREDICTOR_LOG((" marking %s uncacheable", key)); + flags &= ~FLAG_PREFETCHABLE; + } + nsCString newValue; + MakeMetadataEntry(hitCount, lastHit, flags, newValue); + entry->SetMetaDataElement(key, newValue.BeginReading()); + break; + } + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::CacheabilityAction::OnMetaDataElement(const char* asciiKey, + const char* asciiValue) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsURIMetadataElement(asciiKey)) { + return NS_OK; + } + + nsCString key, value; + key.AssignASCII(asciiKey); + value.AssignASCII(asciiValue); + mKeysToCheck.AppendElement(key); + mValuesToCheck.AppendElement(value); + + return NS_OK; +} + +} // namespace net +} // namespace mozilla diff --git a/netwerk/base/Predictor.h b/netwerk/base/Predictor.h @@ -0,0 +1,449 @@ +/* vim: set ts=2 sts=2 et sw=2: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_net_Predictor_h +#define mozilla_net_Predictor_h + +#include "nsINetworkPredictor.h" +#include "nsINetworkPredictorVerifier.h" + +#include "nsCOMPtr.h" +#include "nsICacheEntry.h" +#include "nsICacheEntryOpenCallback.h" +#include "nsICacheStorageService.h" +#include "nsICacheStorageVisitor.h" +#include "nsIDNSListener.h" +#include "nsIInterfaceRequestor.h" +#include "nsIObserver.h" +#include "nsISpeculativeConnect.h" +#include "nsIStreamListener.h" +#include "mozilla/RefPtr.h" +#include "nsString.h" +#include "nsTArray.h" + +#include "mozilla/TimeStamp.h" + +class nsICacheStorage; +class nsIDNSService; +class nsIIOService; +class nsILoadContextInfo; +class nsITimer; + +namespace mozilla { +namespace net { + +class nsHttpRequestHead; +class nsHttpResponseHead; + +class Predictor final : public nsINetworkPredictor, + public nsIObserver, + public nsISpeculativeConnectionOverrider, + public nsIInterfaceRequestor, + public nsICacheEntryMetaDataVisitor, + public nsINetworkPredictorVerifier { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSINETWORKPREDICTOR + NS_DECL_NSIOBSERVER + NS_DECL_NSISPECULATIVECONNECTIONOVERRIDER + NS_DECL_NSIINTERFACEREQUESTOR + NS_DECL_NSICACHEENTRYMETADATAVISITOR + NS_DECL_NSINETWORKPREDICTORVERIFIER + + Predictor(); + + nsresult Init(); + void Shutdown(); + static nsresult Create(const nsIID& iid, void** result); + + // Used to update whether a particular URI was cacheable or not. + // sourceURI and targetURI are the same as the arguments to Learn + // and httpStatus is the status code we got while loading targetURI. + static void UpdateCacheability(nsIURI* sourceURI, nsIURI* targetURI, + uint32_t httpStatus, + nsHttpRequestHead& requestHead, + nsHttpResponseHead* responseHead, + nsILoadContextInfo* lci, bool isTracking); + + private: + virtual ~Predictor(); + + // Stores callbacks for a child process predictor (for test purposes) + nsCOMPtr<nsINetworkPredictorVerifier> mChildVerifier; + + union Reason { + PredictorLearnReason mLearn; + PredictorPredictReason mPredict; + }; + + class DNSListener : public nsIDNSListener { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIDNSLISTENER + + DNSListener() = default; + + private: + virtual ~DNSListener() = default; + }; + + class Action : public nsICacheEntryOpenCallback { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICACHEENTRYOPENCALLBACK + + Action(bool fullUri, bool predict, Reason reason, nsIURI* targetURI, + nsIURI* sourceURI, nsINetworkPredictorVerifier* verifier, + Predictor* predictor); + Action(bool fullUri, bool predict, Reason reason, nsIURI* targetURI, + nsIURI* sourceURI, nsINetworkPredictorVerifier* verifier, + Predictor* predictor, uint8_t stackCount); + + static const bool IS_FULL_URI = true; + static const bool IS_ORIGIN = false; + + static const bool DO_PREDICT = true; + static const bool DO_LEARN = false; + + private: + virtual ~Action() = default; + + bool mFullUri : 1; + bool mPredict : 1; + union { + PredictorPredictReason mPredictReason; + PredictorLearnReason mLearnReason; + }; + nsCOMPtr<nsIURI> mTargetURI; + nsCOMPtr<nsIURI> mSourceURI; + nsCOMPtr<nsINetworkPredictorVerifier> mVerifier; + TimeStamp mStartTime; + uint8_t mStackCount; + RefPtr<Predictor> mPredictor; + }; + + class CacheabilityAction : public nsICacheEntryOpenCallback, + public nsICacheEntryMetaDataVisitor { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICACHEENTRYOPENCALLBACK + NS_DECL_NSICACHEENTRYMETADATAVISITOR + + CacheabilityAction(nsIURI* targetURI, uint32_t httpStatus, + const nsCString& method, bool isTracking, bool couldVary, + bool isNoStore, Predictor* predictor) + : mTargetURI(targetURI), + mHttpStatus(httpStatus), + mMethod(method), + mIsTracking(isTracking), + mCouldVary(couldVary), + mIsNoStore(isNoStore), + mPredictor(predictor) {} + + private: + virtual ~CacheabilityAction() = default; + + nsCOMPtr<nsIURI> mTargetURI; + uint32_t mHttpStatus; + nsCString mMethod; + bool mIsTracking; + bool mCouldVary; + bool mIsNoStore; + RefPtr<Predictor> mPredictor; + nsTArray<nsCString> mKeysToCheck; + nsTArray<nsCString> mValuesToCheck; + }; + + class Resetter : public nsICacheEntryOpenCallback, + public nsICacheEntryMetaDataVisitor, + public nsICacheStorageVisitor { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICACHEENTRYOPENCALLBACK + NS_DECL_NSICACHEENTRYMETADATAVISITOR + NS_DECL_NSICACHESTORAGEVISITOR + + explicit Resetter(Predictor* predictor); + + private: + virtual ~Resetter() = default; + + void Complete(); + + uint32_t mEntriesToVisit; + nsTArray<nsCString> mKeysToDelete; + RefPtr<Predictor> mPredictor; + nsTArray<nsCOMPtr<nsIURI>> mURIsToVisit; + nsTArray<nsCOMPtr<nsILoadContextInfo>> mInfosToVisit; + }; + + class SpaceCleaner : public nsICacheEntryMetaDataVisitor { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSICACHEENTRYMETADATAVISITOR + + explicit SpaceCleaner(Predictor* predictor) + : mLRUStamp(0), mLRUKeyToDelete(nullptr), mPredictor(predictor) {} + + void Finalize(nsICacheEntry* entry); + + private: + virtual ~SpaceCleaner() = default; + uint32_t mLRUStamp; + const char* mLRUKeyToDelete; + nsTArray<nsCString> mLongKeysToDelete; + RefPtr<Predictor> mPredictor; + }; + + class PrefetchListener : public nsIStreamListener { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIREQUESTOBSERVER + NS_DECL_NSISTREAMLISTENER + + PrefetchListener(nsINetworkPredictorVerifier* verifier, nsIURI* uri, + Predictor* predictor) + : mVerifier(verifier), mURI(uri), mPredictor(predictor) {} + + private: + virtual ~PrefetchListener() = default; + + nsCOMPtr<nsINetworkPredictorVerifier> mVerifier; + nsCOMPtr<nsIURI> mURI; + RefPtr<Predictor> mPredictor; + TimeStamp mStartTime; + }; + + // Observer-related stuff + nsresult InstallObserver(); + void RemoveObserver(); + + // Service startup utilities + void MaybeCleanupOldDBFiles(); + + // The guts of prediction + + // This is the top-level driver for doing any prediction that needs + // information from the cache. Returns true if any predictions were queued up + // * reason - What kind of prediction this is/why this prediction is + // happening (pageload, startup) + // * entry - the cache entry with the information we need + // * isNew - whether or not the cache entry is brand new and empty + // * fullUri - whether we are doing predictions based on a full page URI, or + // just the origin of the page + // * targetURI - the URI that we are predicting based upon - IOW, the URI + // that is being loaded or being redirected to + // * verifier - used for testing to verify the expected predictions happen + // * stackCount - used to ensure we don't recurse too far trying to find the + // final redirection in a redirect chain + bool PredictInternal(PredictorPredictReason reason, nsICacheEntry* entry, + bool isNew, bool fullUri, nsIURI* targetURI, + nsINetworkPredictorVerifier* verifier, + uint8_t stackCount); + + // Used when predicting because a page is being loaded (which may include + // being the target of a redirect). All arguments are the same as for + // PredictInternal. Returns true if any predictions were queued up. + bool PredictForPageload(nsICacheEntry* entry, nsIURI* targetURI, + uint8_t stackCount, bool fullUri, + nsINetworkPredictorVerifier* verifier); + + // Used when predicting pages that will be used near browser startup. All + // arguments are the same as for PredictInternal. Returns true if any + // predictions were queued up. + bool PredictForStartup(nsICacheEntry* entry, bool fullUri, + nsINetworkPredictorVerifier* verifier); + + // Utilities related to prediction + + // Used to update our rolling load count (how many of the last n loads was a + // partular resource loaded on?) + // * entry - cache entry of page we're loading + // * flags - value that contains our rolling count as the top 20 bits (but + // we may use fewer than those 20 bits for calculations) + // * key - metadata key that we will update on entry + // * hitCount - part of the metadata we need to preserve + // * lastHit - part of the metadata we need to preserve + void UpdateRollingLoadCount(nsICacheEntry* entry, const uint32_t flags, + const char* key, const uint32_t hitCount, + const uint32_t lastHit); + + // Used to calculate how much to degrade our confidence for all resources + // on a particular page, because of how long ago the most recent load of that + // page was. Returns a value between 0 (very recent most recent load) and 100 + // (very distant most recent load) + // * lastLoad - time stamp of most recent load of a page + int32_t CalculateGlobalDegradation(uint32_t lastLoad); + + // Used to calculate how confident we are that a particular resource will be + // used. Returns a value between 0 (no confidence) and 100 (very confident) + // * hitCount - number of times this resource has been seen when loading + // this page + // * hitsPossible - number of times this page has been loaded + // * lastHit - timestamp of the last time this resource was seen when + // loading this page + // * lastPossible - timestamp of the last time this page was loaded + // * globalDegradation - value calculated by CalculateGlobalDegradation for + // this page + int32_t CalculateConfidence(uint32_t hitCount, uint32_t hitsPossible, + uint32_t lastHit, uint32_t lastPossible, + int32_t globalDegradation); + + // Used to calculate all confidence values for all resources associated with a + // page. + // * entry - the cache entry with all necessary information about this page + // * referrer - the URI that we are loading (may be null) + // * lastLoad - timestamp of the last time this page was loaded + // * loadCount - number of times this page has been loaded + // * gloablDegradation - value calculated by CalculateGlobalDegradation for + // this page + // * fullUri - whether we're predicting for a full URI or origin-only + void CalculatePredictions(nsICacheEntry* entry, nsIURI* referrer, + uint32_t lastLoad, uint32_t loadCount, + int32_t globalDegradation, bool fullUri); + + enum PrefetchIgnoreReason { + PREFETCH_OK, + NOT_FULL_URI, + NO_REFERRER, + MISSED_A_LOAD, + PREFETCH_DISABLED, + PREFETCH_DISABLED_VIA_COUNT, + CONFIDENCE_TOO_LOW + }; + + // Used to prepare any necessary prediction for a resource on a page + // * confidence - value calculated by CalculateConfidence for this resource + // * flags - the flags taken from the resource + // * uri - the ascii spec of the URI of the resource + void SetupPrediction(int32_t confidence, uint32_t flags, const nsCString& uri, + PrefetchIgnoreReason reason); + + // Used to kick off a prefetch from RunPredictions if necessary + // * uri - the URI to prefetch + // * referrer - the URI of the referring page + // * originAttributes - the originAttributes of this prefetch + // * verifier - used for testing to ensure the expected prefetch happens + nsresult Prefetch(nsIURI* uri, nsIURI* referrer, + const OriginAttributes& originAttributes, + nsINetworkPredictorVerifier* verifier); + + // Used to actually perform any predictions set up via SetupPrediction. + // Returns true if any predictions were performed. + // * referrer - the URI we are predicting from + // * originAttributs - the originAttributes we are predicting from + // * verifier - used for testing to ensure the expected predictions happen + bool RunPredictions(nsIURI* referrer, + const OriginAttributes& originAttributes, + nsINetworkPredictorVerifier* verifier); + + // Used to guess whether a page will redirect to another page or not. Returns + // true if a redirection is likely. + // * entry - cache entry with all necessary information about this page + // * loadCount - number of times this page has been loaded + // * lastLoad - timestamp of the last time this page was loaded + // * globalDegradation - value calculated by CalculateGlobalDegradation for + // this page + // * redirectURI - if this returns true, the URI that is likely to be + // redirected to, otherwise null + bool WouldRedirect(nsICacheEntry* entry, uint32_t loadCount, + uint32_t lastLoad, int32_t globalDegradation, + nsIURI** redirectURI); + + // The guts of learning information + + // This is the top-level driver for doing any updating of our information in + // the cache + // * reason - why this learn is happening (pageload, startup, redirect) + // * entry - the cache entry with the information we need + // * isNew - whether or not the cache entry is brand new and empty + // * fullUri - whether we are doing predictions based on a full page URI, or + // just the origin of the page + // * targetURI - the URI that we are adding to our data - most often a + // resource loaded by a page the user navigated to + // * sourceURI - the URI that caused targetURI to be loaded, usually the + // page the user navigated to + void LearnInternal(PredictorLearnReason reason, nsICacheEntry* entry, + bool isNew, bool fullUri, nsIURI* targetURI, + nsIURI* sourceURI); + + // Used when learning about a resource loaded by a page + // * entry - the cache entry with information that needs updating + // * targetURI - the URI of the resource that was loaded by the page + void LearnForSubresource(nsICacheEntry* entry, nsIURI* targetURI); + + // Used when learning about a redirect from one page to another + // * entry - the cache entry of the page that was redirected from + // * targetURI - the URI of the redirect target + void LearnForRedirect(nsICacheEntry* entry, nsIURI* targetURI); + + // Used to learn about pages loaded close to browser startup. This results in + // LearnForStartup being called if we are, in fact, near browser startup + // * uri - the URI of a page that has been loaded (may not have been near + // browser startup) + // * fullUri - true if this is a full page uri, false if it's an origin + // * originAttributes - the originAttributes for this learning. + void MaybeLearnForStartup(nsIURI* uri, bool fullUri, + const OriginAttributes& originAttributes); + + // Used in conjunction with MaybeLearnForStartup to learn about pages loaded + // close to browser startup + // * entry - the cache entry that stores the startup page list + // * targetURI - the URI of a page that was loaded near browser startup + void LearnForStartup(nsICacheEntry* entry, nsIURI* targetURI); + + // Used to parse the data we store in cache metadata + // * key - the cache metadata key + // * value - the cache metadata value + // * uri - (out) the ascii spec of the URI this metadata entry was about + // * hitCount - (out) the number of times this URI has been seen + // * lastHit - (out) timestamp of the last time this URI was seen + // * flags - (out) flags for this metadata entry + bool ParseMetaDataEntry(const char* key, const char* value, nsCString& uri, + uint32_t& hitCount, uint32_t& lastHit, + uint32_t& flags); + + // Used to update whether a particular URI was cacheable or not. + // sourceURI and targetURI are the same as the arguments to Learn + // and httpStatus is the status code we got while loading targetURI. + void UpdateCacheabilityInternal(nsIURI* sourceURI, nsIURI* targetURI, + uint32_t httpStatus, const nsCString& method, + const OriginAttributes& originAttributes, + bool isTracking, bool couldVary, + bool isNoStore); + + // Gets the pref value and clamps it within the acceptable range. + uint32_t ClampedPrefetchRollingLoadCount(); + + // Our state + bool mInitialized{false}; + + nsTArray<nsCString> mKeysToOperateOn; + nsTArray<nsCString> mValuesToOperateOn; + + nsCOMPtr<nsICacheStorageService> mCacheStorageService; + + nsCOMPtr<nsISpeculativeConnect> mSpeculativeService; + + nsCOMPtr<nsIURI> mStartupURI; + uint32_t mStartupTime{0}; + uint32_t mLastStartupTime{0}; + int32_t mStartupCount{1}; + + nsCOMPtr<nsIDNSService> mDnsService; + + RefPtr<DNSListener> mDNSListener; + + nsTArray<nsCOMPtr<nsIURI>> mPrefetches; + nsTArray<nsCOMPtr<nsIURI>> mPreconnects; + nsTArray<nsCOMPtr<nsIURI>> mPreresolves; + + static Predictor* sSelf; +}; + +} // namespace net +} // namespace mozilla + +#endif // mozilla_net_Predictor_h diff --git a/netwerk/base/moz.build b/netwerk/base/moz.build @@ -63,6 +63,8 @@ XPIDL_SOURCES += [ "nsINetworkInfoService.idl", "nsINetworkInterceptController.idl", "nsINetworkLinkService.idl", + "nsINetworkPredictor.idl", + "nsINetworkPredictorVerifier.idl", "nsINullChannel.idl", "nsIParentChannel.idl", "nsIParentRedirectingChannel.idl", @@ -167,6 +169,7 @@ EXPORTS.mozilla.net += [ "InterceptionInfo.h", "IPv4Parser.h", "NetworkConnectivityService.h", + "Predictor.h", "PrivateBrowsingChannel.h", "ProtocolHandlerInfo.h", "RedirectChannelRegistrar.h", @@ -238,6 +241,7 @@ UNIFIED_SOURCES += [ "nsTransportUtils.cpp", "nsUDPSocket.cpp", "PollableEvent.cpp", + "Predictor.cpp", "ProtocolHandlerInfo.cpp", "ProxyAutoConfig.cpp", "RedirectChannelRegistrar.cpp", diff --git a/netwerk/base/nsINetworkPredictor.idl b/netwerk/base/nsINetworkPredictor.idl @@ -0,0 +1,187 @@ +/* vim: set ts=2 sts=2 et sw=2: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIURI; +interface nsINetworkPredictorVerifier; + +webidl Document; + +typedef unsigned long PredictorPredictReason; +typedef unsigned long PredictorLearnReason; + +%{C++ +namespace mozilla { + +class OriginAttributes; + +} +%} + +[ref] native OriginAttributes(const mozilla::OriginAttributes); + +/** + * nsINetworkPredictor - learn about pages users visit, and allow us to take + * predictive actions upon future visits. + * NOTE: nsINetworkPredictor should only + * be used on the main thread. + */ +[scriptable, builtinclass, uuid(acc88e7c-3f39-42c7-ac31-6377c2c3d73e)] +interface nsINetworkPredictor : nsISupports +{ + /** + * Prediction reasons + * + * PREDICT_LOAD - we are being asked to take predictive action because + * the user has initiated a pageload. + * + * PREDICT_STARTUP - we are being asked to take predictive action + * because the browser is starting up. + */ + const PredictorPredictReason PREDICT_LOAD = 1; + const PredictorPredictReason PREDICT_STARTUP = 2; + + /** + * Start taking predictive actions + * + * Calling this will cause the predictor to (possibly) start + * taking actions such as DNS prefetch and/or TCP preconnect based on + * (1) the host name that we are given, and (2) the reason we are being + * asked to take actions. + * + * @param targetURI - The URI we are being asked to take actions based on. + * @param sourceURI - The URI that is currently loaded. This is so we can + * avoid doing predictive actions for link hover on an HTTPS page (for + * example). + * @param reason - The reason we are being asked to take actions. Can be + * any of the PREDICT_* values above. + * In the case of PREDICT_LOAD, targetURI should be the URI of the page that + * is being loaded and sourceURI should be null. + * In the case of PREDICT_STARTUP, both targetURI and sourceURI should be + * null. + * @param originAttributes - The originAttributes of the page load we are + * predicting about. + * @param verifier - An nsINetworkPredictorVerifier used in testing to ensure + * we're predicting the way we expect to. Not necessary (or desired) for + * normal operation. + */ + [implicit_jscontext] + void predict(in nsIURI targetURI, + in nsIURI sourceURI, + in PredictorPredictReason reason, + in jsval originAttributes, + in nsINetworkPredictorVerifier verifier); + + [notxpcom] + nsresult predictNative(in nsIURI targetURI, + in nsIURI sourceURI, + in PredictorPredictReason reason, + in OriginAttributes originAttributes, + in nsINetworkPredictorVerifier verifier); + + + /* + * Reasons we are learning something + * + * LEARN_LOAD_TOPLEVEL - we are learning about the toplevel resource of a + * pageload (NOTE: this should ONLY be used by tests) + * + * LEARN_LOAD_SUBRESOURCE - we are learning a subresource from a pageload + * + * LEARN_LOAD_REDIRECT - we are learning about the re-direct of a URI + * + * LEARN_STARTUP - we are learning about a page loaded during startup + */ + const PredictorLearnReason LEARN_LOAD_TOPLEVEL = 0; + const PredictorLearnReason LEARN_LOAD_SUBRESOURCE = 1; + const PredictorLearnReason LEARN_LOAD_REDIRECT = 2; + const PredictorLearnReason LEARN_STARTUP = 3; + + /** + * Add to our compendium of knowledge + * + * This adds to our prediction database to make things (hopefully) + * smarter next time we predict something. + * + * @param targetURI - The URI that was loaded that we are keeping track of. + * @param sourceURI - The URI that caused targetURI to be loaded (for page + * loads). This means the DOCUMENT URI. + * @param reason - The reason we are learning this bit of knowledge. + * Reason can be any of the LEARN_* values. + * In the case of LEARN_LOAD_SUBRESOURCE, targetURI should be the URI of a + * subresource of a page, and sourceURI should be the top-level URI. + * In the case of LEARN_LOAD_REDIRECT, targetURI is the NEW URI of a + * top-level resource that was redirected to, and sourceURI is the + * ORIGINAL URI of said top-level resource. + * In the case of LEARN_STARTUP, targetURI should be the URI of a page + * that was loaded immediately after browser startup, and sourceURI should + * be null. + * @param originAttributes - The originAttributes for the page load that we + * are learning about. + */ + [implicit_jscontext] + void learn(in nsIURI targetURI, + in nsIURI sourceURI, + in PredictorLearnReason reason, + in jsval originAttributes); + + [notxpcom] + nsresult learnNative(in nsIURI targetURI, + in nsIURI sourceURI, + in PredictorLearnReason reason, + in OriginAttributes originAttributes); + + /** + * Clear out all our learned knowledge + * + * This removes everything from our database so that any predictions begun + * after this completes will start from a blank slate. + */ + void reset(); +}; + +%{C++ +// Wrapper functions to make use of the predictor easier and less invasive +class nsIChannel; + +class nsILoadContext; +class nsILoadGroup; +class nsINetworkPredictorVerifier; + +namespace mozilla { + +class OriginAttributes; + +namespace net { + +nsresult PredictorPredict(nsIURI *targetURI, + nsIURI *sourceURI, + PredictorPredictReason reason, + const OriginAttributes& originAttributes, + nsINetworkPredictorVerifier *verifier); + +nsresult PredictorLearn(nsIURI *targetURI, + nsIURI *sourceURI, + PredictorLearnReason reason, + const OriginAttributes& originAttributes); + +nsresult PredictorLearn(nsIURI *targetURI, + nsIURI *sourceURI, + PredictorLearnReason reason, + nsILoadGroup *loadGroup); + +nsresult PredictorLearn(nsIURI *targetURI, + nsIURI *sourceURI, + PredictorLearnReason reason, + dom::Document *document); + +nsresult PredictorLearnRedirect(nsIURI *targetURI, + nsIChannel *channel, + const OriginAttributes& originAttributes); + +} // mozilla::net +} // mozilla +%} diff --git a/netwerk/base/nsINetworkPredictorVerifier.idl b/netwerk/base/nsINetworkPredictorVerifier.idl @@ -0,0 +1,40 @@ +/* vim: set ts=2 sts=2 et sw=2: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * nsINetworkPredictorVerifier - used for testing the network predictor to + * ensure it does what we expect it to do. + */ + +#include "nsISupports.idl" + +interface nsIURI; + +[scriptable, uuid(2e43bb32-dabf-4494-9f90-2b3195b1c73d)] +interface nsINetworkPredictorVerifier : nsISupports +{ + /** + * Callback for when we do a predictive prefetch + * + * @param uri - The URI that was prefetched + * @param status - The request status code returned by the + * prefetch attempt e.g. 200 (OK) + */ + void onPredictPrefetch(in nsIURI uri, in uint32_t status); + + /** + * Callback for when we do a predictive preconnect + * + * @param uri - The URI that was preconnected to + */ + void onPredictPreconnect(in nsIURI uri); + + /** + * Callback for when we do a predictive DNS lookup + * + * @param uri - The URI that was looked up + */ + void onPredictDNS(in nsIURI uri); +}; diff --git a/netwerk/base/nsISpeculativeConnect.idl b/netwerk/base/nsISpeculativeConnect.idl @@ -85,6 +85,12 @@ interface nsISpeculativeConnectionOverrider : nsISupports */ [infallible] readonly attribute boolean ignoreIdle; + /* + * Used by the Predictor to gather telemetry data on speculative connection + * usage. + */ + [infallible] readonly attribute boolean isFromPredictor; + /** * by default speculative connections are not made to RFC 1918 addresses */ diff --git a/netwerk/base/nsUDPSocket.cpp b/netwerk/base/nsUDPSocket.cpp @@ -3,6 +3,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "Predictor.h" #include "mozilla/Components.h" #include "mozilla/dom/TypedArray.h" #include "mozilla/HoldDropJSObjects.h" diff --git a/netwerk/build/components.conf b/netwerk/build/components.conf @@ -260,6 +260,14 @@ Classes = [ 'headers': ['mozilla/net/NetworkConnectivityService.h'], }, { + 'name': 'Predictor', + 'cid': '{969adfdf-7221-4419-aecf-05f8faf00c9b}', + 'contract_ids': ['@mozilla.org/network/predictor;1'], + 'singleton': True, + 'legacy_constructor': 'mozilla::net::Predictor::Create', + 'headers': ['mozilla/net/Predictor.h'], + }, + { 'name': 'ProtocolProxy', 'cid': '{e9b301c0-e0e4-11d3-a1a8-0050041caf44}', 'contract_ids': ['@mozilla.org/network/protocol-proxy-service;1'], diff --git a/netwerk/build/nsNetCID.h b/netwerk/build/nsNetCID.h @@ -394,6 +394,15 @@ */ #define NS_URICLASSIFIERSERVICE_CONTRACTID "@mozilla.org/uriclassifierservice" +// service implementing nsINetworkPredictor +#define NS_NETWORKPREDICTOR_CONTRACTID "@mozilla.org/network/predictor;1" +#define NS_NETWORKPREDICTOR_CID \ + {/* {969adfdf-7221-4419-aecf-05f8faf00c9b} */ \ + 0x969adfdf, \ + 0x7221, \ + 0x4419, \ + {0xae, 0xcf, 0x05, 0xf8, 0xfa, 0xf0, 0x0c, 0x9b}} + // captive portal service implementing nsICaptivePortalService #define NS_CAPTIVEPORTAL_CONTRACTID \ "@mozilla.org/network/captive-portal-service;1" diff --git a/netwerk/cache2/CacheStorageService.cpp b/netwerk/cache2/CacheStorageService.cpp @@ -19,6 +19,7 @@ #include "nsIObserverService.h" #include "nsIFile.h" #include "nsIURI.h" +#include "nsINetworkPredictor.h" #include "nsCOMPtr.h" #include "nsContentUtils.h" #include "nsNetCID.h" diff --git a/netwerk/ipc/NeckoChannelParams.ipdlh b/netwerk/ipc/NeckoChannelParams.ipdlh @@ -648,6 +648,7 @@ struct TransactionObserverResult struct SpeculativeConnectionOverriderArgs { uint32_t parallelSpeculativeConnectLimit; bool ignoreIdle; + bool isFromPredictor; bool allow1918; }; diff --git a/netwerk/ipc/NeckoChild.cpp b/netwerk/ipc/NeckoChild.cpp @@ -33,6 +33,8 @@ #include "SerializedLoadContext.h" #include "nsGlobalWindowInner.h" #include "nsIOService.h" +#include "nsINetworkPredictor.h" +#include "nsINetworkPredictorVerifier.h" #include "nsINetworkLinkService.h" #include "nsQueryObject.h" #include "mozilla/ipc/URIUtils.h" @@ -256,6 +258,60 @@ bool NeckoChild::DeallocPTransportProviderChild( return true; } +/* Predictor Messages */ +mozilla::ipc::IPCResult NeckoChild::RecvPredOnPredictPrefetch( + nsIURI* aURI, const uint32_t& aHttpStatus) { + MOZ_ASSERT(NS_IsMainThread(), + "PredictorChild::RecvOnPredictPrefetch " + "off main thread."); + if (!aURI) { + return IPC_FAIL(this, "aURI is null"); + } + + // Get the current predictor + nsresult rv = NS_OK; + nsCOMPtr<nsINetworkPredictorVerifier> predictor; + predictor = mozilla::components::Predictor::Service(&rv); + NS_ENSURE_SUCCESS(rv, IPC_FAIL_NO_REASON(this)); + + predictor->OnPredictPrefetch(aURI, aHttpStatus); + return IPC_OK(); +} + +mozilla::ipc::IPCResult NeckoChild::RecvPredOnPredictPreconnect(nsIURI* aURI) { + MOZ_ASSERT(NS_IsMainThread(), + "PredictorChild::RecvOnPredictPreconnect " + "off main thread."); + if (!aURI) { + return IPC_FAIL(this, "aURI is null"); + } + // Get the current predictor + nsresult rv = NS_OK; + nsCOMPtr<nsINetworkPredictorVerifier> predictor; + predictor = mozilla::components::Predictor::Service(&rv); + NS_ENSURE_SUCCESS(rv, IPC_FAIL_NO_REASON(this)); + + predictor->OnPredictPreconnect(aURI); + return IPC_OK(); +} + +mozilla::ipc::IPCResult NeckoChild::RecvPredOnPredictDNS(nsIURI* aURI) { + MOZ_ASSERT(NS_IsMainThread(), + "PredictorChild::RecvOnPredictDNS off " + "main thread."); + if (!aURI) { + return IPC_FAIL(this, "aURI is null"); + } + // Get the current predictor + nsresult rv = NS_OK; + nsCOMPtr<nsINetworkPredictorVerifier> predictor; + predictor = mozilla::components::Predictor::Service(&rv); + NS_ENSURE_SUCCESS(rv, IPC_FAIL_NO_REASON(this)); + + predictor->OnPredictDNS(aURI); + return IPC_OK(); +} + mozilla::ipc::IPCResult NeckoChild::RecvSpeculativeConnectRequest() { nsCOMPtr<nsIObserverService> obsService = services::GetObserverService(); if (obsService) { diff --git a/netwerk/ipc/NeckoParent.cpp b/netwerk/ipc/NeckoParent.cpp @@ -53,6 +53,8 @@ #include "nsEscape.h" #include "SerializedLoadContext.h" #include "nsAuthInformationHolder.h" +#include "nsINetworkPredictor.h" +#include "nsINetworkPredictorVerifier.h" #include "nsISpeculativeConnect.h" #include "nsFileChannel.h" #include "nsHttpHandler.h" @@ -576,6 +578,49 @@ bool NeckoParent::DeallocPTransportProviderParent( return true; } +/* Predictor Messages */ +mozilla::ipc::IPCResult NeckoParent::RecvPredPredict( + nsIURI* aTargetURI, nsIURI* aSourceURI, const uint32_t& aReason, + const OriginAttributes& aOriginAttributes, const bool& hasVerifier) { + // Get the current predictor + nsresult rv = NS_OK; + nsCOMPtr<nsINetworkPredictor> predictor; + predictor = mozilla::components::Predictor::Service(&rv); + NS_ENSURE_SUCCESS(rv, IPC_OK()); + + nsCOMPtr<nsINetworkPredictorVerifier> verifier; + if (hasVerifier) { + verifier = do_QueryInterface(predictor); + } + predictor->PredictNative(aTargetURI, aSourceURI, aReason, aOriginAttributes, + verifier); + return IPC_OK(); +} + +mozilla::ipc::IPCResult NeckoParent::RecvPredLearn( + nsIURI* aTargetURI, nsIURI* aSourceURI, const uint32_t& aReason, + const OriginAttributes& aOriginAttributes) { + // Get the current predictor + nsresult rv = NS_OK; + nsCOMPtr<nsINetworkPredictor> predictor; + predictor = mozilla::components::Predictor::Service(&rv); + NS_ENSURE_SUCCESS(rv, IPC_OK()); + + predictor->LearnNative(aTargetURI, aSourceURI, aReason, aOriginAttributes); + return IPC_OK(); +} + +mozilla::ipc::IPCResult NeckoParent::RecvPredReset() { + // Get the current predictor + nsresult rv = NS_OK; + nsCOMPtr<nsINetworkPredictor> predictor; + predictor = mozilla::components::Predictor::Service(&rv); + NS_ENSURE_SUCCESS(rv, IPC_OK()); + + predictor->Reset(); + return IPC_OK(); +} + mozilla::ipc::IPCResult NeckoParent::RecvRequestContextLoadBegin( const uint64_t& rcid) { nsCOMPtr<nsIRequestContextService> rcsvc = diff --git a/netwerk/ipc/NeckoParent.h b/netwerk/ipc/NeckoParent.h @@ -9,6 +9,7 @@ #include "mozilla/net/PNeckoParent.h" #include "mozilla/net/NeckoCommon.h" #include "nsIAuthPrompt2.h" +#include "nsINetworkPredictor.h" #include "nsNetUtil.h" #ifndef mozilla_net_NeckoParent_h @@ -174,6 +175,18 @@ class NeckoParent : public PNeckoParent { PTransportProviderParent* AllocPTransportProviderParent(); bool DeallocPTransportProviderParent(PTransportProviderParent* aActor); + /* Predictor Messages */ + mozilla::ipc::IPCResult RecvPredPredict( + nsIURI* aTargetURI, nsIURI* aSourceURI, + const PredictorPredictReason& aReason, + const OriginAttributes& aOriginAttributes, const bool& hasVerifier); + + mozilla::ipc::IPCResult RecvPredLearn( + nsIURI* aTargetURI, nsIURI* aSourceURI, + const PredictorPredictReason& aReason, + const OriginAttributes& aOriginAttributes); + mozilla::ipc::IPCResult RecvPredReset(); + mozilla::ipc::IPCResult RecvRequestContextLoadBegin(const uint64_t& rcid); mozilla::ipc::IPCResult RecvRequestContextAfterDOMContentLoaded( const uint64_t& rcid); diff --git a/netwerk/ipc/PNecko.ipdl b/netwerk/ipc/PNecko.ipdl @@ -93,6 +93,15 @@ parent: async PWebSocketEventListener(uint64_t aInnerWindowID); + /* Predictor Methods */ + async PredPredict(nullable nsIURI targetURI, nullable nsIURI sourceURI, + uint32_t reason, OriginAttributes originAttributes, + bool hasVerifier); + [Priority=low] + async PredLearn(nullable nsIURI targetURI, nullable nsIURI sourceURI, + uint32_t reason, OriginAttributes originAttributes); + async PredReset(); + async SpeculativeConnect(nullable nsIURI uri, nullable nsIPrincipal principal, OriginAttributes? originAttributes, @@ -159,6 +168,11 @@ parent: async GetPageIconStream(nullable nsIURI uri, LoadInfoArgs loadInfo) returns (RemoteStreamInfo? info); child: + /* Predictor Methods */ + async PredOnPredictPrefetch(nullable nsIURI uri, uint32_t httpStatus); + async PredOnPredictPreconnect(nullable nsIURI uri); + async PredOnPredictDNS(nullable nsIURI uri); + async SpeculativeConnectRequest(); // Using medium high priority to deliver this notification possibly sooner than we diff --git a/netwerk/protocol/http/AlternateServices.cpp b/netwerk/protocol/http/AlternateServices.cpp @@ -1393,6 +1393,12 @@ AltSvcOverride::GetParallelSpeculativeConnectLimit( } NS_IMETHODIMP +AltSvcOverride::GetIsFromPredictor(bool* isFromPredictor) { + *isFromPredictor = false; + return NS_OK; +} + +NS_IMETHODIMP AltSvcOverride::GetAllow1918(bool* allow) { // normally we don't do speculative connects to 1918.. and we use // speculative connects for the mapping validation, so override diff --git a/netwerk/protocol/http/ConnectionEntry.cpp b/netwerk/protocol/http/ConnectionEntry.cpp @@ -1067,14 +1067,14 @@ bool ConnectionEntry::MaybeProcessCoalescingKeys(nsIDNSAddrRecord* dnsRecord, nsresult ConnectionEntry::CreateDnsAndConnectSocket( nsAHttpTransaction* trans, uint32_t caps, bool speculative, - bool urgentStart, bool allow1918, + bool isFromPredictor, bool urgentStart, bool allow1918, PendingTransactionInfo* pendingTransInfo) { MOZ_ASSERT(OnSocketThread(), "not on socket thread"); MOZ_ASSERT((speculative && !pendingTransInfo) || (!speculative && pendingTransInfo)); - RefPtr<DnsAndConnectSocket> sock = - new DnsAndConnectSocket(mConnInfo, trans, caps, speculative, urgentStart); + RefPtr<DnsAndConnectSocket> sock = new DnsAndConnectSocket( + mConnInfo, trans, caps, speculative, isFromPredictor, urgentStart); if (speculative) { sock->SetAllow1918(allow1918); diff --git a/netwerk/protocol/http/ConnectionEntry.h b/netwerk/protocol/http/ConnectionEntry.h @@ -118,8 +118,8 @@ class ConnectionEntry : public SupportsWeakPtr { bool aIsHttp3 = false); nsresult CreateDnsAndConnectSocket(nsAHttpTransaction* trans, uint32_t caps, - bool speculative, bool urgentStart, - bool allow1918, + bool speculative, bool isFromPredictor, + bool urgentStart, bool allow1918, PendingTransactionInfo* pendingTransInfo); // Spdy sometimes resolves the address in the socket manager in order diff --git a/netwerk/protocol/http/DnsAndConnectSocket.cpp b/netwerk/protocol/http/DnsAndConnectSocket.cpp @@ -64,11 +64,12 @@ static void NotifyActivity(nsHttpConnectionInfo* aConnInfo, uint32_t aSubtype) { DnsAndConnectSocket::DnsAndConnectSocket(nsHttpConnectionInfo* ci, nsAHttpTransaction* trans, uint32_t caps, bool speculative, - bool urgentStart) + bool isFromPredictor, bool urgentStart) : mTransaction(trans), mCaps(caps), mSpeculative(speculative), mUrgentStart(urgentStart), + mIsFromPredictor(isFromPredictor), mConnInfo(ci) { MOZ_ASSERT(ci && trans, "constructor with null arguments"); LOG(("Creating DnsAndConnectSocket [this=%p trans=%p ent=%s key=%s]\n", this, diff --git a/netwerk/protocol/http/DnsAndConnectSocket.h b/netwerk/protocol/http/DnsAndConnectSocket.h @@ -49,7 +49,8 @@ class DnsAndConnectSocket final : public nsIOutputStreamCallback, NS_DECL_NSIDNSLISTENER DnsAndConnectSocket(nsHttpConnectionInfo* ci, nsAHttpTransaction* trans, - uint32_t caps, bool speculative, bool urgentStart); + uint32_t caps, bool speculative, bool isFromPredictor, + bool urgentStart); [[nodiscard]] nsresult Init(ConnectionEntry* ent); void Abandon(); @@ -235,6 +236,11 @@ class DnsAndConnectSocket final : public nsIOutputStreamCallback, // mark the connection as urgent rightaway it's created. bool mUrgentStart; + // mIsFromPredictor is set if the socket originated from the network + // Predictor. It is used to gather telemetry data on used speculative + // connections from the predictor. + bool mIsFromPredictor; + bool mAllow1918 = true; // mHasConnected tracks whether one of the sockets has completed the diff --git a/netwerk/protocol/http/EarlyHintPreconnect.cpp b/netwerk/protocol/http/EarlyHintPreconnect.cpp @@ -57,6 +57,12 @@ EarlyHintsPreConnectOverride::GetParallelSpeculativeConnectLimit( } NS_IMETHODIMP +EarlyHintsPreConnectOverride::GetIsFromPredictor(bool* isFromPredictor) { + *isFromPredictor = false; + return NS_OK; +} + +NS_IMETHODIMP EarlyHintsPreConnectOverride::GetAllow1918(bool* allow) { *allow = false; return NS_OK; diff --git a/netwerk/protocol/http/HttpConnectionMgrChild.cpp b/netwerk/protocol/http/HttpConnectionMgrChild.cpp @@ -144,6 +144,12 @@ SpeculativeConnectionOverrider::GetParallelSpeculativeConnectLimit( } NS_IMETHODIMP +SpeculativeConnectionOverrider::GetIsFromPredictor(bool* aIsFromPredictor) { + *aIsFromPredictor = mArgs.isFromPredictor(); + return NS_OK; +} + +NS_IMETHODIMP SpeculativeConnectionOverrider::GetAllow1918(bool* aAllow) { *aAllow = mArgs.allow1918(); return NS_OK; diff --git a/netwerk/protocol/http/HttpConnectionMgrParent.cpp b/netwerk/protocol/http/HttpConnectionMgrParent.cpp @@ -235,6 +235,7 @@ nsresult HttpConnectionMgrParent::SpeculativeConnect( overriderArgs->parallelSpeculativeConnectLimit() = overrider->GetParallelSpeculativeConnectLimit(); overriderArgs->ignoreIdle() = overrider->GetIgnoreIdle(); + overriderArgs->isFromPredictor() = overrider->GetIsFromPredictor(); overriderArgs->allow1918() = overrider->GetAllow1918(); } diff --git a/netwerk/protocol/http/SpeculativeTransaction.cpp b/netwerk/protocol/http/SpeculativeTransaction.cpp @@ -28,6 +28,7 @@ SpeculativeTransaction::CreateWithNewConnInfo(nsHttpConnectionInfo* aConnInfo) { new SpeculativeTransaction(aConnInfo, mCallbacks, mCaps); trans->mParallelSpeculativeConnectLimit = mParallelSpeculativeConnectLimit; trans->mIgnoreIdle = mIgnoreIdle; + trans->mIsFromPredictor = mIsFromPredictor; trans->mAllow1918 = mAllow1918; return trans.forget(); } diff --git a/netwerk/protocol/http/SpeculativeTransaction.h b/netwerk/protocol/http/SpeculativeTransaction.h @@ -34,12 +34,16 @@ class SpeculativeTransaction : public NullHttpTransaction { mParallelSpeculativeConnectLimit.emplace(aLimit); } void SetIgnoreIdle(bool aIgnoreIdle) { mIgnoreIdle.emplace(aIgnoreIdle); } + void SetIsFromPredictor(bool aIsFromPredictor) { + mIsFromPredictor.emplace(aIsFromPredictor); + } void SetAllow1918(bool aAllow1918) { mAllow1918.emplace(aAllow1918); } const Maybe<uint32_t>& ParallelSpeculativeConnectLimit() { return mParallelSpeculativeConnectLimit; } const Maybe<bool>& IgnoreIdle() { return mIgnoreIdle; } + const Maybe<bool>& IsFromPredictor() { return mIsFromPredictor; } const Maybe<bool>& Allow1918() { return mAllow1918; } void Close(nsresult aReason) override; @@ -52,6 +56,7 @@ class SpeculativeTransaction : public NullHttpTransaction { Maybe<uint32_t> mParallelSpeculativeConnectLimit; Maybe<bool> mIgnoreIdle; + Maybe<bool> mIsFromPredictor; Maybe<bool> mAllow1918; bool mTriedToWrite = false; diff --git a/netwerk/protocol/http/nsHttpChannel.cpp b/netwerk/protocol/http/nsHttpChannel.cpp @@ -115,6 +115,7 @@ #include "nsCORSListenerProxy.h" #include "nsISocketProvider.h" #include "mozilla/extensions/StreamFilterParent.h" +#include "mozilla/net/Predictor.h" #include "mozilla/net/SFVService.h" #include "mozilla/NullPrincipal.h" #include "CacheControlParser.h" @@ -3034,6 +3035,20 @@ nsresult nsHttpChannel::ProcessResponse(nsHttpConnectionInfo* aConnInfo) { // We use GetReferringPage because mReferrerInfo may not be set at all(this is // especially useful in xpcshell tests, where we don't have an actual pageload // to get a referrer from). + if (StaticPrefs::network_predictor_enabled()) { + nsCOMPtr<nsIURI> referrer = GetReferringPage(); + if (!referrer && mReferrerInfo) { + referrer = mReferrerInfo->GetOriginalReferrer(); + } + + if (referrer) { + nsCOMPtr<nsILoadContextInfo> lci = GetLoadContextInfo(this); + mozilla::net::Predictor::UpdateCacheability( + referrer, mURI, httpStatus, mRequestHead, mResponseHead.get(), lci, + IsThirdPartyTrackingResource()); + } + } + // Only allow 407 (authentication required) to continue if (mTransaction && mTransaction->ProxyConnectFailed() && httpStatus != 407) { return ProcessFailedProxyConnect(httpStatus); diff --git a/netwerk/protocol/http/nsHttpConnectionMgr.cpp b/netwerk/protocol/http/nsHttpConnectionMgr.cpp @@ -529,6 +529,7 @@ nsresult nsHttpConnectionMgr::SpeculativeConnect( args->mTrans->SetParallelSpeculativeConnectLimit( overrider->GetParallelSpeculativeConnectLimit()); args->mTrans->SetIgnoreIdle(overrider->GetIgnoreIdle()); + args->mTrans->SetIsFromPredictor(overrider->GetIsFromPredictor()); args->mTrans->SetAllow1918(overrider->GetAllow1918()); } @@ -1341,7 +1342,7 @@ nsresult nsHttpConnectionMgr::MakeNewConnection( } nsresult rv = ent->CreateDnsAndConnectSocket( - trans, trans->Caps(), false, + trans, trans->Caps(), false, false, trans->GetClassOfService().Flags() & nsIClassOfService::UrgentStart, true, pendingTransInfo); if (NS_FAILED(rv)) { @@ -3585,6 +3586,8 @@ void nsHttpConnectionMgr::DoSpeculativeConnectionInternal( ? *aTrans->ParallelSpeculativeConnectLimit() : gHttpHandler->ParallelSpeculativeConnectLimit(); bool ignoreIdle = aTrans->IgnoreIdle() ? *aTrans->IgnoreIdle() : false; + bool isFromPredictor = + aTrans->IsFromPredictor() ? *aTrans->IsFromPredictor() : false; bool allow1918 = aTrans->Allow1918() ? *aTrans->Allow1918() : false; bool keepAlive = aTrans->Caps() & NS_HTTP_ALLOW_KEEPALIVE; @@ -3595,7 +3598,8 @@ void nsHttpConnectionMgr::DoSpeculativeConnectionInternal( !(keepAlive && aEnt->RestrictConnections()) && !AtActiveConnectionLimit(aEnt, aTrans->Caps())) { nsresult rv = aEnt->CreateDnsAndConnectSocket(aTrans, aTrans->Caps(), true, - false, allow1918, nullptr); + isFromPredictor, false, + allow1918, nullptr); if (NS_FAILED(rv)) { LOG( ("DoSpeculativeConnectionInternal Transport socket creation " diff --git a/netwerk/test/unit/test_predictor.js b/netwerk/test/unit/test_predictor.js @@ -0,0 +1,819 @@ +"use strict"; + +const { HttpServer } = ChromeUtils.importESModule( + "resource://testing-common/httpd.sys.mjs" +); +const ReferrerInfo = Components.Constructor( + "@mozilla.org/referrer-info;1", + "nsIReferrerInfo", + "init" +); + +var running_single_process = false; + +var predictor = null; + +function is_child_process() { + return Services.appinfo.processType == Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT; +} + +function extract_origin(uri) { + var o = uri.scheme + "://" + uri.asciiHost; + if (uri.port !== -1) { + o = o + ":" + uri.port; + } + return o; +} + +var origin_attributes = {}; + +var ValidityChecker = function (verifier, httpStatus) { + this.verifier = verifier; + this.httpStatus = httpStatus; +}; + +ValidityChecker.prototype = { + verifier: null, + httpStatus: 0, + + QueryInterface: ChromeUtils.generateQI(["nsICacheEntryOpenCallback"]), + + onCacheEntryCheck() { + return Ci.nsICacheEntryOpenCallback.ENTRY_WANTED; + }, + + onCacheEntryAvailable(entry) { + // Check if forced valid + Assert.equal(entry.isForcedValid, this.httpStatus === 200); + this.verifier.maybe_run_next_test(); + }, +}; + +var Verifier = function _verifier( + testing, + expected_prefetches, + expected_preconnects, + expected_preresolves +) { + this.verifying = testing; + this.expected_prefetches = expected_prefetches; + this.expected_preconnects = expected_preconnects; + this.expected_preresolves = expected_preresolves; +}; + +Verifier.prototype = { + complete: false, + verifying: null, + expected_prefetches: null, + expected_preconnects: null, + expected_preresolves: null, + + getInterface: function verifier_getInterface(iid) { + return this.QueryInterface(iid); + }, + + QueryInterface: ChromeUtils.generateQI(["nsINetworkPredictorVerifier"]), + + maybe_run_next_test: function verifier_maybe_run_next_test() { + if ( + this.expected_prefetches.length === 0 && + this.expected_preconnects.length === 0 && + this.expected_preresolves.length === 0 && + !this.complete + ) { + this.complete = true; + Assert.ok(true, "Well this is unexpected..."); + // This kicks off the ability to run the next test + reset_predictor(); + } + }, + + onPredictPrefetch: function verifier_onPredictPrefetch(uri, status) { + var index = this.expected_prefetches.indexOf(uri.asciiSpec); + if (index == -1 && !this.complete) { + Assert.ok(false, "Got prefetch for unexpected uri " + uri.asciiSpec); + } else { + this.expected_prefetches.splice(index, 1); + } + + dump("checking validity of entry for " + uri.spec + "\n"); + var checker = new ValidityChecker(this, status); + asyncOpenCacheEntry( + uri.spec, + "disk", + Ci.nsICacheStorage.OPEN_NORMALLY, + Services.loadContextInfo.default, + checker + ); + }, + + onPredictPreconnect: function verifier_onPredictPreconnect(uri) { + var origin = extract_origin(uri); + var index = this.expected_preconnects.indexOf(origin); + if (index == -1 && !this.complete) { + Assert.ok(false, "Got preconnect for unexpected uri " + origin); + } else { + this.expected_preconnects.splice(index, 1); + } + this.maybe_run_next_test(); + }, + + onPredictDNS: function verifier_onPredictDNS(uri) { + var origin = extract_origin(uri); + var index = this.expected_preresolves.indexOf(origin); + if (index == -1 && !this.complete) { + Assert.ok(false, "Got preresolve for unexpected uri " + origin); + } else { + this.expected_preresolves.splice(index, 1); + } + this.maybe_run_next_test(); + }, +}; + +function reset_predictor() { + if (running_single_process || is_child_process()) { + predictor.reset(); + } else { + sendCommand("predictor.reset();"); + } +} + +function newURI(s) { + return Services.io.newURI(s); +} + +var prepListener = { + numEntriesToOpen: 0, + numEntriesOpened: 0, + continueCallback: null, + + QueryInterface: ChromeUtils.generateQI(["nsICacheEntryOpenCallback"]), + + init(entriesToOpen, cb) { + this.numEntriesOpened = 0; + this.numEntriesToOpen = entriesToOpen; + this.continueCallback = cb; + }, + + onCacheEntryCheck() { + return Ci.nsICacheEntryOpenCallback.ENTRY_WANTED; + }, + + onCacheEntryAvailable(entry, isNew, result) { + Assert.equal(result, Cr.NS_OK); + entry.setMetaDataElement("predictor_test", "1"); + entry.metaDataReady(); + this.numEntriesOpened++; + if (this.numEntriesToOpen == this.numEntriesOpened) { + this.continueCallback(); + } + }, +}; + +function open_and_continue(uris, continueCallback) { + var ds = Services.cache2.diskCacheStorage(Services.loadContextInfo.default); + + prepListener.init(uris.length, continueCallback); + for (var i = 0; i < uris.length; ++i) { + ds.asyncOpenURI( + uris[i], + "", + Ci.nsICacheStorage.OPEN_NORMALLY, + prepListener + ); + } +} + +const pageload_toplevel = newURI("http://localhost:4444/index.html"); + +function continue_test_pageload() { + var subresources = [ + "http://localhost:4444/style.css", + "http://localhost:4443/jquery.js", + "http://localhost:4444/image.png", + ]; + + // This is necessary to learn the origin stuff + predictor.learn( + pageload_toplevel, + null, + predictor.LEARN_LOAD_TOPLEVEL, + origin_attributes + ); + do_timeout(0, () => { + // allow the learn() to run on the main thread + var preconns = []; + + var sruri = newURI(subresources[0]); + predictor.learn( + sruri, + pageload_toplevel, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + preconns.push(extract_origin(sruri)); + + sruri = newURI(subresources[1]); + predictor.learn( + sruri, + pageload_toplevel, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + preconns.push(extract_origin(sruri)); + + sruri = newURI(subresources[2]); + predictor.learn( + sruri, + pageload_toplevel, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + preconns.push(extract_origin(sruri)); + + var verifier = new Verifier("pageload", [], preconns, []); + predictor.predict( + pageload_toplevel, + null, + predictor.PREDICT_LOAD, + origin_attributes, + verifier + ); + }); + }); + }); + }); +} + +function test_pageload() { + open_and_continue([pageload_toplevel], function () { + if (running_single_process) { + continue_test_pageload(); + } else { + sendCommand("continue_test_pageload();"); + } + }); +} + +const redirect_inituri = newURI("http://localhost:4443/redirect"); +const redirect_targeturi = newURI("http://localhost:4444/index.html"); + +function continue_test_redirect() { + var subresources = [ + "http://localhost:4444/style.css", + "http://localhost:4443/jquery.js", + "http://localhost:4444/image.png", + ]; + + predictor.learn( + redirect_inituri, + null, + predictor.LEARN_LOAD_TOPLEVEL, + origin_attributes + ); + do_timeout(0, () => { + predictor.learn( + redirect_targeturi, + null, + predictor.LEARN_LOAD_TOPLEVEL, + origin_attributes + ); + do_timeout(0, () => { + predictor.learn( + redirect_targeturi, + redirect_inituri, + predictor.LEARN_LOAD_REDIRECT, + origin_attributes + ); + do_timeout(0, () => { + var preconns = []; + preconns.push(extract_origin(redirect_targeturi)); + + var sruri = newURI(subresources[0]); + predictor.learn( + sruri, + redirect_targeturi, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + preconns.push(extract_origin(sruri)); + + sruri = newURI(subresources[1]); + predictor.learn( + sruri[1], + redirect_targeturi, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + preconns.push(extract_origin(sruri)); + + sruri = newURI(subresources[2]); + predictor.learn( + sruri[2], + redirect_targeturi, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + preconns.push(extract_origin(sruri)); + + var verifier = new Verifier("redirect", [], preconns, []); + predictor.predict( + redirect_inituri, + null, + predictor.PREDICT_LOAD, + origin_attributes, + verifier + ); + }); + }); + }); + }); + }); + }); +} + +// Test is currently disabled. +// eslint-disable-next-line no-unused-vars +function test_redirect() { + open_and_continue([redirect_inituri, redirect_targeturi], function () { + if (running_single_process) { + continue_test_redirect(); + } else { + sendCommand("continue_test_redirect();"); + } + }); +} + +// Test is currently disabled. +// eslint-disable-next-line no-unused-vars +function test_startup() { + if (!running_single_process && !is_child_process()) { + // This one we can just proxy to the child and be done with, no extra setup + // is necessary. + sendCommand("test_startup();"); + return; + } + + var uris = ["http://localhost:4444/startup", "http://localhost:4443/startup"]; + var preconns = []; + var uri = newURI(uris[0]); + predictor.learn(uri, null, predictor.LEARN_STARTUP, origin_attributes); + do_timeout(0, () => { + preconns.push(extract_origin(uri)); + + uri = newURI(uris[1]); + predictor.learn(uri, null, predictor.LEARN_STARTUP, origin_attributes); + do_timeout(0, () => { + preconns.push(extract_origin(uri)); + + var verifier = new Verifier("startup", [], preconns, []); + predictor.predict( + null, + null, + predictor.PREDICT_STARTUP, + origin_attributes, + verifier + ); + }); + }); +} + +const dns_toplevel = newURI("http://localhost:4444/index.html"); + +function continue_test_dns() { + var subresource = "http://localhost:4443/jquery.js"; + + predictor.learn( + dns_toplevel, + null, + predictor.LEARN_LOAD_TOPLEVEL, + origin_attributes + ); + do_timeout(0, () => { + var sruri = newURI(subresource); + predictor.learn( + sruri, + dns_toplevel, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + var preresolves = [extract_origin(sruri)]; + var verifier = new Verifier("dns", [], [], preresolves); + predictor.predict( + dns_toplevel, + null, + predictor.PREDICT_LOAD, + origin_attributes, + verifier + ); + }); + }); +} + +function test_dns() { + open_and_continue([dns_toplevel], function () { + // Ensure that this will do preresolves + Services.prefs.setIntPref( + "network.predictor.preconnect-min-confidence", + 101 + ); + if (running_single_process) { + continue_test_dns(); + } else { + sendCommand("continue_test_dns();"); + } + }); +} + +const origin_toplevel = newURI("http://localhost:4444/index.html"); + +function continue_test_origin() { + var subresources = [ + "http://localhost:4444/style.css", + "http://localhost:4443/jquery.js", + "http://localhost:4444/image.png", + ]; + predictor.learn( + origin_toplevel, + null, + predictor.LEARN_LOAD_TOPLEVEL, + origin_attributes + ); + do_timeout(0, () => { + var preconns = []; + + var sruri = newURI(subresources[0]); + predictor.learn( + sruri, + origin_toplevel, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + var origin = extract_origin(sruri); + if (!preconns.includes(origin)) { + preconns.push(origin); + } + + sruri = newURI(subresources[1]); + predictor.learn( + sruri, + origin_toplevel, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + var origin1 = extract_origin(sruri); + if (!preconns.includes(origin1)) { + preconns.push(origin1); + } + + sruri = newURI(subresources[2]); + predictor.learn( + sruri, + origin_toplevel, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + do_timeout(0, () => { + var origin2 = extract_origin(sruri); + if (!preconns.includes(origin2)) { + preconns.push(origin2); + } + + var loaduri = newURI("http://localhost:4444/anotherpage.html"); + var verifier = new Verifier("origin", [], preconns, []); + predictor.predict( + loaduri, + null, + predictor.PREDICT_LOAD, + origin_attributes, + verifier + ); + }); + }); + }); + }); +} + +function test_origin() { + open_and_continue([origin_toplevel], function () { + if (running_single_process) { + continue_test_origin(); + } else { + sendCommand("continue_test_origin();"); + } + }); +} + +var httpserv = null; +var prefetch_tluri; +var prefetch_sruri; + +function prefetchHandler(metadata, response) { + response.setStatusLine(metadata.httpVersion, 200, "OK"); + var body = "Success (meow meow meow)."; + + response.bodyOutputStream.write(body, body.length); +} + +var prefetchListener = { + onStartRequest(request) { + Assert.equal(request.status, Cr.NS_OK); + }, + + onDataAvailable(request, stream, offset, cnt) { + read_stream(stream, cnt); + }, + + onStopRequest() { + run_next_test(); + }, +}; + +function test_prefetch_setup() { + // Disable preconnects and preresolves + Services.prefs.setIntPref("network.predictor.preconnect-min-confidence", 101); + Services.prefs.setIntPref("network.predictor.preresolve-min-confidence", 101); + + Services.prefs.setBoolPref("network.predictor.enable-prefetch", true); + + // Makes it so we only have to call test_prefetch_prime twice to make prefetch + // do its thing. + Services.prefs.setIntPref("network.predictor.prefetch-rolling-load-count", 2); + + // This test does not run in e10s-mode, so we'll just go ahead and skip it. + // We've left the e10s test code in below, just in case someone wants to try + // to make it work at some point in the future. + if (!running_single_process) { + dump("skipping test_prefetch_setup due to e10s\n"); + run_next_test(); + return; + } + + httpserv = new HttpServer(); + httpserv.registerPathHandler("/cat.jpg", prefetchHandler); + httpserv.start(-1); + + var tluri = + "http://127.0.0.1:" + httpserv.identity.primaryPort + "/index.html"; + var sruri = "http://127.0.0.1:" + httpserv.identity.primaryPort + "/cat.jpg"; + prefetch_tluri = newURI(tluri); + prefetch_sruri = newURI(sruri); + if (!running_single_process && !is_child_process()) { + // Give the child process access to these values + sendCommand('prefetch_tluri = newURI("' + tluri + '");'); + sendCommand('prefetch_sruri = newURI("' + sruri + '");'); + } + + run_next_test(); +} + +// Used to "prime the pump" for prefetch - it makes sure all our learns go +// through as expected so that prefetching will happen. +function test_prefetch_prime() { + // This test does not run in e10s-mode, so we'll just go ahead and skip it. + // We've left the e10s test code in below, just in case someone wants to try + // to make it work at some point in the future. + if (!running_single_process) { + dump("skipping test_prefetch_prime due to e10s\n"); + run_next_test(); + return; + } + + open_and_continue([prefetch_tluri], function () { + if (running_single_process) { + predictor.learn( + prefetch_tluri, + null, + predictor.LEARN_LOAD_TOPLEVEL, + origin_attributes + ); + predictor.learn( + prefetch_sruri, + prefetch_tluri, + predictor.LEARN_LOAD_SUBRESOURCE, + origin_attributes + ); + } else { + sendCommand( + "predictor.learn(prefetch_tluri, null, predictor.LEARN_LOAD_TOPLEVEL, origin_attributes);" + ); + sendCommand( + "predictor.learn(prefetch_sruri, prefetch_tluri, predictor.LEARN_LOAD_SUBRESOURCE, origin_attributes);" + ); + } + + // This runs in the parent or only process + var channel = NetUtil.newChannel({ + uri: prefetch_sruri.asciiSpec, + loadUsingSystemPrincipal: true, + }).QueryInterface(Ci.nsIHttpChannel); + channel.requestMethod = "GET"; + channel.referrerInfo = new ReferrerInfo( + Ci.nsIReferrerInfo.EMPTY, + true, + prefetch_tluri + ); + channel.asyncOpen(prefetchListener); + }); +} + +function test_prefetch() { + // This test does not run in e10s-mode, so we'll just go ahead and skip it. + // We've left the e10s test code in below, just in case someone wants to try + // to make it work at some point in the future. + if (!running_single_process) { + dump("skipping test_prefetch due to e10s\n"); + run_next_test(); + return; + } + + // Setup for this has all been taken care of by test_prefetch_prime, so we can + // continue on without pausing here. + if (running_single_process) { + continue_test_prefetch(); + } else { + sendCommand("continue_test_prefetch();"); + } +} + +function continue_test_prefetch() { + var prefetches = [prefetch_sruri.asciiSpec]; + var verifier = new Verifier("prefetch", prefetches, [], []); + predictor.predict( + prefetch_tluri, + null, + predictor.PREDICT_LOAD, + origin_attributes, + verifier + ); +} + +function test_visitor_doom() { + // See bug 1708673 + Services.prefs.setBoolPref("network.cache.bug1708673", true); + registerCleanupFunction(() => { + Services.prefs.clearUserPref("network.cache.bug1708673"); + }); + + let p1 = new Promise(resolve => { + let doomTasks = []; + let visitor = { + onCacheStorageInfo() {}, + async onCacheEntryInfo( + aURI, + aIdEnhance, + aDataSize, + aAltDataSize, + aFetchCount, + aLastModifiedTime, + aExpirationTime, + aPinned, + aInfo + ) { + let storages = [ + Services.cache2.memoryCacheStorage(aInfo), + Services.cache2.diskCacheStorage(aInfo, false), + ]; + console.debug("asyncDoomURI", aURI.spec); + let doomTask = Promise.all( + storages.map(storage => { + return new Promise(resolve1 => { + storage.asyncDoomURI(aURI, aIdEnhance, { + onCacheEntryDoomed: resolve1, + }); + }); + }) + ); + doomTasks.push(doomTask); + }, + onCacheEntryVisitCompleted() { + Promise.allSettled(doomTasks).then(resolve); + }, + QueryInterface: ChromeUtils.generateQI(["nsICacheStorageVisitor"]), + }; + Services.cache2.asyncVisitAllStorages(visitor, true); + }); + + let p2 = new Promise(resolve => { + reset_predictor(); + resolve(); + }); + + do_test_pending(); + Promise.allSettled([p1, p2]).then(() => { + return new Promise(resolve => { + let entryCount = 0; + let visitor = { + onCacheStorageInfo() {}, + async onCacheEntryInfo() { + entryCount++; + }, + onCacheEntryVisitCompleted() { + Assert.equal(entryCount, 0); + resolve(); + }, + QueryInterface: ChromeUtils.generateQI(["nsICacheStorageVisitor"]), + }; + Services.cache2.asyncVisitAllStorages(visitor, true); + }).then(run_next_test); + }); +} + +function cleanup() { + observer.cleaningUp = true; + if (running_single_process) { + // The http server is required (and started) by the prefetch test, which + // only runs in single-process mode, so don't try to shut it down if we're + // in e10s mode. + do_test_pending(); + httpserv.stop(do_test_finished); + } + reset_predictor(); +} + +var tests = [ + // This must ALWAYS come first, to ensure a clean slate + reset_predictor, + test_pageload, + // TODO: These are disabled until the features are re-written + //test_redirect, + //test_startup, + // END DISABLED TESTS + test_origin, + test_dns, + test_prefetch_setup, + test_prefetch_prime, + test_prefetch_prime, + test_prefetch, + test_visitor_doom, + // This must ALWAYS come last, to ensure we clean up after ourselves + cleanup, +]; + +var observer = { + cleaningUp: false, + + QueryInterface: ChromeUtils.generateQI(["nsIObserver"]), + + observe(subject, topic) { + if (topic != "predictor-reset-complete") { + return; + } + + if (this.cleaningUp) { + unregisterObserver(); + } + + run_next_test(); + }, +}; + +function registerObserver() { + Services.obs.addObserver(observer, "predictor-reset-complete"); +} + +function unregisterObserver() { + Services.obs.removeObserver(observer, "predictor-reset-complete"); +} + +function run_test_real() { + tests.forEach(f => add_test(f)); + do_get_profile(); + + Services.prefs.setBoolPref("network.predictor.enabled", true); + Services.prefs.setBoolPref("network.predictor.doing-tests", true); + + predictor = Cc["@mozilla.org/network/predictor;1"].getService( + Ci.nsINetworkPredictor + ); + + registerObserver(); + + registerCleanupFunction(() => { + Services.prefs.clearUserPref("network.predictor.preconnect-min-confidence"); + Services.prefs.clearUserPref("network.predictor.enabled"); + Services.prefs.clearUserPref("network.predictor.preresolve-min-confidence"); + Services.prefs.clearUserPref("network.predictor.enable-prefetch"); + Services.prefs.clearUserPref( + "network.predictor.prefetch-rolling-load-count" + ); + Services.prefs.clearUserPref("network.predictor.doing-tests"); + }); + + run_next_test(); +} + +function run_test() { + // This indirection is necessary to make e10s tests work as expected + running_single_process = true; + run_test_real(); +} diff --git a/netwerk/test/unit/xpcshell.toml b/netwerk/test/unit/xpcshell.toml @@ -1216,6 +1216,8 @@ skip-if = [ ["test_post.js"] +["test_predictor.js"] + ["test_prio_header_override_forbid.js"] ["test_private_cookie_changed.js"] diff --git a/netwerk/test/unit_ipc/test_predictor_wrap.js b/netwerk/test/unit_ipc/test_predictor_wrap.js @@ -0,0 +1,44 @@ +// This is the bit that runs in the parent process when the test begins. Here's +// what happens: +// +// - Load the entire single-process test in the parent +// - Setup the test harness within the child process +// - Send a command to the child to have the single-process test loaded there, as well +// - Send a command to the child to make the predictor available +// - run_test_real in the parent +// - run_test_real does a bunch of pref-setting and other fun things on the parent +// - once all that's done, it calls run_next_test IN THE PARENT +// - every time run_next_test is called, it is called IN THE PARENT +// - the test that gets started then does any parent-side setup that's necessary +// - once the parent-side setup is done, it calls continue_<testname> IN THE CHILD +// - when the test is done running on the child, the child calls predictor.reset +// this causes some code to be run on the parent which, when complete, sends an +// obserer service notification IN THE PARENT, which causes run_next_test to be +// called again, bumping us up to the top of the loop outlined here +// - when the final test is done, cleanup happens IN THE PARENT and we're done +// +// This is a little confusing, but it's what we have to do in order to have some +// things that must run on the parent (the setup - opening cache entries, etc) +// but with most of the test running on the child (calls to the predictor api, +// verification, etc). +// +/* import-globals-from ../unit/test_predictor.js */ + +function run_test() { + var test_path = do_get_file("../unit/test_predictor.js").path.replace( + /\\/g, + "/" + ); + load(test_path); + do_load_child_test_harness(); + do_test_pending(); + sendCommand('load("' + test_path + '");', function () { + sendCommand( + 'predictor = Cc["@mozilla.org/network/predictor;1"].getService(Ci.nsINetworkPredictor);', + function () { + run_test_real(); + do_test_finished(); + } + ); + }); +} diff --git a/netwerk/test/unit_ipc/xpcshell.toml b/netwerk/test/unit_ipc/xpcshell.toml @@ -22,6 +22,7 @@ support-files = [ "!/netwerk/test/unit/test_headers.js", "!/netwerk/test/unit/test_httpsuspend.js", "!/netwerk/test/unit/test_post.js", + "!/netwerk/test/unit/test_predictor.js", "!/netwerk/test/unit/test_progress.js", "!/netwerk/test/unit/test_redirect_veto.js", "!/netwerk/test/unit/test_redirect-caching_canceled.js", @@ -193,6 +194,9 @@ prefs = ["network.allow_raw_sockets_in_content_processes=true"] ["test_post_wrap.js"] prefs = ["network.allow_raw_sockets_in_content_processes=true"] +["test_predictor_wrap.js"] +run-sequentially = ["os == 'mac'"] # frequent timeouts on Mac in parallel + ["test_progress_wrap.js"] prefs = ["network.allow_raw_sockets_in_content_processes=true"]