tor-browser

URLPreloader.cpp (20353B)

---

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "ScriptPreloader-inl.h"
#include "mozilla/URLPreloader.h"
#include "mozilla/loader/AutoMemMap.h"

#include "mozilla/ClearOnShutdown.h"
#include "mozilla/FileUtils.h"
#include "mozilla/IOBuffers.h"
#include "mozilla/Logging.h"
#include "mozilla/ScopeExit.h"
#include "mozilla/Services.h"
#include "mozilla/Try.h"
#include "mozilla/Vector.h"
#include "mozilla/scache/StartupCache.h"

#include "crc32c.h"
#include "MainThreadUtils.h"
#include "nsPrintfCString.h"
#include "nsDebug.h"
#include "nsIFile.h"
#include "nsIFileURL.h"
#include "nsNetUtil.h"
#include "nsPromiseFlatString.h"
#include "nsProxyRelease.h"
#include "nsThreadUtils.h"
#include "nsXULAppAPI.h"
#include "nsZipArchive.h"
#include "xpcpublic.h"

namespace mozilla {
namespace {
static LazyLogModule gURLLog("URLPreloader");

#define LOG(level, ...) MOZ_LOG(gURLLog, LogLevel::level, (__VA_ARGS__))

template <typename T>
bool StartsWith(const T& haystack, const T& needle) {
 return StringHead(haystack, needle.Length()) == needle;
}
}  // anonymous namespace

using namespace mozilla::loader;
using mozilla::scache::StartupCache;

nsresult URLPreloader::CollectReports(nsIHandleReportCallback* aHandleReport,
                                     nsISupports* aData, bool aAnonymize) {
 MOZ_COLLECT_REPORT("explicit/url-preloader/other", KIND_HEAP, UNITS_BYTES,
                    ShallowSizeOfIncludingThis(MallocSizeOf),
                    "Memory used by the URL preloader service itself.");

 for (const auto& elem : mCachedURLs.Values()) {
   nsAutoCString pathName;
   pathName.Append(elem->mPath);
   // The backslashes will automatically be replaced with slashes in
   // about:memory, without splitting each path component into a separate
   // branch in the memory report tree.
   pathName.ReplaceChar('/', '\\');

   nsPrintfCString path("explicit/url-preloader/cached-urls/%s/[%s]",
                        elem->TypeString(), pathName.get());

   aHandleReport->Callback(
       ""_ns, path, KIND_HEAP, UNITS_BYTES,
       elem->SizeOfIncludingThis(MallocSizeOf),
       nsLiteralCString("Memory used to hold cache data for files which "
                        "have been read or pre-loaded during this session."),
       aData);
 }

 return NS_OK;
}

// static
already_AddRefed<URLPreloader> URLPreloader::Create(bool* aInitialized) {
 // The static APIs like URLPreloader::Read work in the child process because
 // they fall back to a synchronous read. The actual preloader must be
 // explicitly initialized, and this should only be done in the parent.
 MOZ_RELEASE_ASSERT(XRE_IsParentProcess());

 RefPtr<URLPreloader> preloader = new URLPreloader();
 if (preloader->InitInternal().isOk()) {
   *aInitialized = true;
   RegisterWeakMemoryReporter(preloader);
 } else {
   *aInitialized = false;
 }

 return preloader.forget();
}

URLPreloader& URLPreloader::GetSingleton() {
 if (!sSingleton) {
   sSingleton = Create(&sInitialized);
   ClearOnShutdown(&sSingleton);
 }

 return *sSingleton;
}

bool URLPreloader::sInitialized = false;

StaticRefPtr<URLPreloader> URLPreloader::sSingleton;

URLPreloader::~URLPreloader() {
 if (sInitialized) {
   UnregisterWeakMemoryReporter(this);
   sInitialized = false;
 }
}

Result<Ok, nsresult> URLPreloader::InitInternal() {
 MOZ_RELEASE_ASSERT(NS_IsMainThread());

 if (Omnijar::HasOmnijar(Omnijar::GRE)) {
   MOZ_TRY(Omnijar::GetURIString(Omnijar::GRE, mGREPrefix));
 }
 if (Omnijar::HasOmnijar(Omnijar::APP)) {
   MOZ_TRY(Omnijar::GetURIString(Omnijar::APP, mAppPrefix));
 }

 nsresult rv;
 nsCOMPtr<nsIIOService> ios = do_GetIOService(&rv);
 MOZ_TRY(rv);

 nsCOMPtr<nsIProtocolHandler> ph;
 MOZ_TRY(ios->GetProtocolHandler("resource", getter_AddRefs(ph)));

 mResProto = do_QueryInterface(ph, &rv);
 MOZ_TRY(rv);

 mChromeReg = services::GetChromeRegistry();
 if (!mChromeReg) {
   return Err(NS_ERROR_UNEXPECTED);
 }

 MOZ_TRY(NS_GetSpecialDirectory("ProfLDS", getter_AddRefs(mProfD)));

 return Ok();
}

URLPreloader& URLPreloader::ReInitialize() {
 MOZ_ASSERT(sSingleton);
 sSingleton = nullptr;
 sSingleton = Create(&sInitialized);
 return *sSingleton;
}

Result<nsCOMPtr<nsIFile>, nsresult> URLPreloader::GetCacheFile(
   const nsAString& suffix) {
 if (!mProfD) {
   return Err(NS_ERROR_NOT_INITIALIZED);
 }

 nsCOMPtr<nsIFile> cacheFile;
 MOZ_TRY(mProfD->Clone(getter_AddRefs(cacheFile)));

 MOZ_TRY(cacheFile->AppendNative("startupCache"_ns));
 (void)cacheFile->Create(nsIFile::DIRECTORY_TYPE, 0777);

 MOZ_TRY(cacheFile->Append(u"urlCache"_ns + suffix));

 return std::move(cacheFile);
}

static const uint8_t URL_MAGIC[] = "mozURLcachev003";

Result<nsCOMPtr<nsIFile>, nsresult> URLPreloader::FindCacheFile() {
 if (StartupCache::GetIgnoreDiskCache()) {
   return Err(NS_ERROR_ABORT);
 }

 nsCOMPtr<nsIFile> cacheFile = MOZ_TRY(GetCacheFile(u".bin"_ns));

 bool exists;
 MOZ_TRY(cacheFile->Exists(&exists));
 if (exists) {
   MOZ_TRY(cacheFile->MoveTo(nullptr, u"urlCache-current.bin"_ns));
 } else {
   MOZ_TRY(cacheFile->SetLeafName(u"urlCache-current.bin"_ns));
   MOZ_TRY(cacheFile->Exists(&exists));
   if (!exists) {
     return Err(NS_ERROR_FILE_NOT_FOUND);
   }
 }

 return std::move(cacheFile);
}

Result<Ok, nsresult> URLPreloader::WriteCache() {
 MOZ_ASSERT(!NS_IsMainThread());
 MOZ_DIAGNOSTIC_ASSERT(mStartupFinished);

 // The script preloader might call us a second time, if it has to re-write
 // its cache after a cache flush. We don't care about cache flushes, since
 // our cache doesn't store any file data, only paths. And we currently clear
 // our cached file list after the first write, which means that a second
 // write would (aside from breaking the invariant that we never touch
 // mCachedURLs off-main-thread after the first write, and trigger a data
 // race) mean we get no pre-loading on the next startup.
 if (mCacheWritten) {
   return Ok();
 }
 mCacheWritten = true;

 LOG(Debug, "Writing cache...");

 nsCOMPtr<nsIFile> cacheFile = MOZ_TRY(GetCacheFile(u"-new.bin"_ns));

 bool exists;
 MOZ_TRY(cacheFile->Exists(&exists));
 if (exists) {
   MOZ_TRY(cacheFile->Remove(false));
 }

 {
   AutoFDClose raiiFd;
   MOZ_TRY(cacheFile->OpenNSPRFileDesc(PR_WRONLY | PR_CREATE_FILE, 0644,
                                       getter_Transfers(raiiFd)));
   const auto fd = raiiFd.get();

   nsTArray<URLEntry*> entries;
   for (const auto& entry : mCachedURLs.Values()) {
     if (entry->mReadTime) {
       entries.AppendElement(entry.get());
     }
   }

   entries.Sort(URLEntry::Comparator());

   OutputBuffer buf;
   for (auto entry : entries) {
     entry->Code(buf);
   }

   uint8_t headerSize[4];
   LittleEndian::writeUint32(headerSize, buf.cursor());

   uint8_t crc[4];
   LittleEndian::writeUint32(crc, ComputeCrc32c(~0, buf.Get(), buf.cursor()));

   MOZ_TRY(Write(fd, URL_MAGIC, sizeof(URL_MAGIC)));
   MOZ_TRY(Write(fd, headerSize, sizeof(headerSize)));
   MOZ_TRY(Write(fd, crc, sizeof(crc)));
   MOZ_TRY(Write(fd, buf.Get(), buf.cursor()));
 }

 MOZ_TRY(cacheFile->MoveTo(nullptr, u"urlCache.bin"_ns));

 NS_DispatchToMainThread(
     NewRunnableMethod("URLPreloader::Cleanup", this, &URLPreloader::Cleanup));

 return Ok();
}

void URLPreloader::Cleanup() { mCachedURLs.Clear(); }

Result<Ok, nsresult> URLPreloader::ReadCache(
   LinkedList<URLEntry>& pendingURLs) {
 LOG(Debug, "Reading cache...");

 nsCOMPtr<nsIFile> cacheFile = MOZ_TRY(FindCacheFile());

 AutoMemMap cache;
 MOZ_TRY(cache.init(cacheFile));

 auto size = cache.size();

 uint32_t headerSize;
 uint32_t crc;
 if (size < sizeof(URL_MAGIC) + sizeof(headerSize) + sizeof(crc)) {
   return Err(NS_ERROR_UNEXPECTED);
 }

 auto data = cache.get<uint8_t>();
 auto end = data + size;

 if (memcmp(URL_MAGIC, data.get(), sizeof(URL_MAGIC))) {
   return Err(NS_ERROR_UNEXPECTED);
 }
 data += sizeof(URL_MAGIC);

 headerSize = LittleEndian::readUint32(data.get());
 data += sizeof(headerSize);

 crc = LittleEndian::readUint32(data.get());
 data += sizeof(crc);

 if (data + headerSize > end) {
   return Err(NS_ERROR_UNEXPECTED);
 }

 if (crc != ComputeCrc32c(~0, data.get(), headerSize)) {
   return Err(NS_ERROR_UNEXPECTED);
 }

 {
   mMonitor.AssertCurrentThreadOwns();

   auto cleanup = MakeScopeExit([&]() {
     while (auto* elem = pendingURLs.getFirst()) {
       elem->remove();
     }
     mCachedURLs.Clear();
   });

   Range<const uint8_t> header(data, data + headerSize);
   data += headerSize;

   InputBuffer buf(header);
   while (!buf.finished()) {
     CacheKey key(buf);

     LOG(Debug, "Cached file: %s %s", key.TypeString(), key.mPath.get());

     // Don't bother doing anything else if the key didn't load correctly.
     // We're going to throw it out right away, and it is possible that this
     // leads to pendingURLs getting into a weird state.
     if (buf.error()) {
       return Err(NS_ERROR_UNEXPECTED);
     }

     auto entry = mCachedURLs.GetOrInsertNew(key, key);
     entry->mResultCode = NS_ERROR_NOT_INITIALIZED;

     if (entry->isInList()) {
#ifdef NIGHTLY_BUILD
       MOZ_DIAGNOSTIC_ASSERT(pendingURLs.contains(entry),
                             "Entry should be in pendingURLs");
       MOZ_DIAGNOSTIC_ASSERT(key.mPath.Length() > 0,
                             "Path should be non-empty");
       MOZ_DIAGNOSTIC_CRASH("Entry should be new and not in any list");
#endif
       return Err(NS_ERROR_UNEXPECTED);
     }

     pendingURLs.insertBack(entry);
   }

   MOZ_RELEASE_ASSERT(!buf.error(),
                      "We should have already bailed on an error");

   cleanup.release();
 }

 return Ok();
}

void URLPreloader::BackgroundReadFiles() {
 auto cleanup = MakeScopeExit([&]() {
   auto lock = mReaderThread.Lock();
   auto& readerThread = lock.ref();
   NS_DispatchToMainThread(NewRunnableMethod(
       "nsIThread::AsyncShutdown", readerThread, &nsIThread::AsyncShutdown));

   readerThread = nullptr;
 });

 Vector<nsZipCursor> cursors;
 LinkedList<URLEntry> pendingURLs;
 {
   MonitorAutoLock mal(mMonitor);

   if (ReadCache(pendingURLs).isErr()) {
     mReaderInitialized = true;
     mal.NotifyAll();
     return;
   }

   int numZipEntries = 0;
   for (auto entry : pendingURLs) {
     if (entry->mType != entry->TypeFile) {
       numZipEntries++;
     }
   }
   MOZ_RELEASE_ASSERT(cursors.reserve(numZipEntries));

   // Initialize the zip cursors for all files in Omnijar while the monitor
   // is locked. Omnijar is not threadsafe, so the caller of
   // AutoBeginReading guard must ensure that no code accesses Omnijar
   // until this segment is done. Once the cursors have been initialized,
   // the actual reading and decompression can safely be done off-thread,
   // as is the case for thread-retargeted jar: channels.
   for (auto entry : pendingURLs) {
     if (entry->mType == entry->TypeFile) {
       continue;
     }

     RefPtr<nsZipArchive> zip = entry->Archive();
     if (!zip) {
       MOZ_CRASH_UNSAFE_PRINTF(
           "Failed to get Omnijar %s archive for entry (path: \"%s\")",
           entry->TypeString(), entry->mPath.get());
     }

     auto item = zip->GetItem(entry->mPath);
     if (!item) {
       entry->mResultCode = NS_ERROR_FILE_NOT_FOUND;
       continue;
     }

     size_t size = item->RealSize();

     entry->mData.SetLength(size);
     auto data = entry->mData.BeginWriting();

     cursors.infallibleEmplaceBack(item, zip, reinterpret_cast<uint8_t*>(data),
                                   size, true);
   }

   mReaderInitialized = true;
   mal.NotifyAll();
 }

 // Loop over the entries, read the file's contents, store them in the
 // entry's mData pointer, and notify any waiting threads to check for
 // completion.
 uint32_t i = 0;
 for (auto entry : pendingURLs) {
   // If there is any other error code, the entry has already failed at
   // this point, so don't bother trying to read it again.
   if (entry->mResultCode != NS_ERROR_NOT_INITIALIZED) {
     continue;
   }

   nsresult rv = NS_OK;

   LOG(Debug, "Background reading %s file %s", entry->TypeString(),
       entry->mPath.get());

   if (entry->mType == entry->TypeFile) {
     auto result = entry->Read();
     if (result.isErr()) {
       rv = result.unwrapErr();
     }
   } else {
     auto& cursor = cursors[i++];

     uint32_t len;
     cursor.Copy(&len);
     if (len != entry->mData.Length()) {
       entry->mData.Truncate();
       rv = NS_ERROR_FAILURE;
     }
   }

   entry->mResultCode = rv;
   mMonitor.NotifyAll();
 }

 // We're done reading pending entries, so clear the list.
 pendingURLs.clear();
}

void URLPreloader::BeginBackgroundRead() {
 auto lock = mReaderThread.Lock();
 auto& readerThread = lock.ref();
 if (!readerThread && !mReaderInitialized && sInitialized) {
   nsresult rv;
   rv = NS_NewNamedThread("BGReadURLs", getter_AddRefs(readerThread));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return;
   }

   nsCOMPtr<nsIRunnable> runnable =
       NewRunnableMethod("URLPreloader::BackgroundReadFiles", this,
                         &URLPreloader::BackgroundReadFiles);
   rv = readerThread->Dispatch(runnable.forget(), NS_DISPATCH_NORMAL);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     // If we can't launch the task, just destroy the thread
     readerThread = nullptr;
     return;
   }
 }
}

Result<nsCString, nsresult> URLPreloader::ReadInternal(const CacheKey& key,
                                                      ReadType readType) {
 if (mStartupFinished || !mReaderInitialized) {
   URLEntry entry(key);

   return entry.Read();
 }

 auto entry = mCachedURLs.GetOrInsertNew(key, key);

 entry->UpdateUsedTime();

 return entry->ReadOrWait(readType);
}

Result<nsCString, nsresult> URLPreloader::ReadURIInternal(nsIURI* uri,
                                                         ReadType readType) {
 CacheKey key = MOZ_TRY(ResolveURI(uri));

 return ReadInternal(key, readType);
}

/* static */ Result<nsCString, nsresult> URLPreloader::Read(const CacheKey& key,
                                                           ReadType readType) {
 // If we're being called before the preloader has been initialized (i.e.,
 // before the profile has been initialized), just fall back to a synchronous
 // read. This happens when we're reading .ini and preference files that are
 // needed to locate and initialize the profile.
 if (!sInitialized) {
   return URLEntry(key).Read();
 }

 return GetSingleton().ReadInternal(key, readType);
}

/* static */ Result<nsCString, nsresult> URLPreloader::ReadURI(
   nsIURI* uri, ReadType readType) {
 if (!sInitialized) {
   return Err(NS_ERROR_NOT_INITIALIZED);
 }

 return GetSingleton().ReadURIInternal(uri, readType);
}

/* static */ Result<nsCString, nsresult> URLPreloader::ReadFile(
   nsIFile* file, ReadType readType) {
 return Read(CacheKey(file), readType);
}

/* static */ Result<nsCString, nsresult> URLPreloader::Read(
   FileLocation& location, ReadType readType) {
 if (location.IsZip()) {
   if (location.GetBaseZip()) {
     nsCString path;
     location.GetPath(path);
     return ReadZip(location.GetBaseZip(), path);
   }
   return URLEntry::ReadLocation(location);
 }

 nsCOMPtr<nsIFile> file = location.GetBaseFile();
 return ReadFile(file, readType);
}

/* static */ Result<nsCString, nsresult> URLPreloader::ReadZip(
   nsZipArchive* zip, const nsACString& path, ReadType readType) {
 // If the zip archive belongs to an Omnijar location, map it to a cache
 // entry, and cache it as normal. Otherwise, simply read the entry
 // synchronously, since other JAR archives are currently unsupported by the
 // cache.
 RefPtr<nsZipArchive> reader = Omnijar::GetReader(Omnijar::GRE);
 if (zip == reader) {
   CacheKey key(CacheKey::TypeGREJar, path);
   return Read(key, readType);
 }

 reader = Omnijar::GetReader(Omnijar::APP);
 if (zip == reader) {
   CacheKey key(CacheKey::TypeAppJar, path);
   return Read(key, readType);
 }

 // Not an Omnijar archive, so just read it directly.
 FileLocation location(zip, path);
 return URLEntry::ReadLocation(location);
}

Result<URLPreloader::CacheKey, nsresult> URLPreloader::ResolveURI(nsIURI* uri) {
 nsCString spec;
 nsCString scheme;
 MOZ_TRY(uri->GetSpec(spec));
 MOZ_TRY(uri->GetScheme(scheme));

 nsCOMPtr<nsIURI> resolved;

 // If the URI is a resource: or chrome: URI, first resolve it to the
 // underlying URI that it wraps.
 if (scheme.EqualsLiteral("resource")) {
   MOZ_TRY(mResProto->ResolveURI(uri, spec));
   MOZ_TRY(NS_NewURI(getter_AddRefs(resolved), spec));
 } else if (scheme.EqualsLiteral("chrome")) {
   MOZ_TRY(mChromeReg->ConvertChromeURL(uri, getter_AddRefs(resolved)));
   MOZ_TRY(resolved->GetSpec(spec));
 } else {
   resolved = uri;
 }
 MOZ_TRY(resolved->GetScheme(scheme));

 // Try the GRE and App Omnijar prefixes.
 if (mGREPrefix.Length() && StartsWith(spec, mGREPrefix)) {
   return CacheKey(CacheKey::TypeGREJar, Substring(spec, mGREPrefix.Length()));
 }

 if (mAppPrefix.Length() && StartsWith(spec, mAppPrefix)) {
   return CacheKey(CacheKey::TypeAppJar, Substring(spec, mAppPrefix.Length()));
 }

 // Try for a file URI.
 if (scheme.EqualsLiteral("file")) {
   nsCOMPtr<nsIFileURL> fileURL = do_QueryInterface(resolved);
   MOZ_ASSERT(fileURL);

   nsCOMPtr<nsIFile> file;
   MOZ_TRY(fileURL->GetFile(getter_AddRefs(file)));

   nsString path;
   MOZ_TRY(file->GetPath(path));

   return CacheKey(CacheKey::TypeFile, NS_ConvertUTF16toUTF8(path));
 }

 // Not a file or Omnijar URI, so currently unsupported.
 return Err(NS_ERROR_INVALID_ARG);
}

size_t URLPreloader::ShallowSizeOfIncludingThis(
   mozilla::MallocSizeOf mallocSizeOf) {
 return (mallocSizeOf(this) +
         mAppPrefix.SizeOfExcludingThisEvenIfShared(mallocSizeOf) +
         mGREPrefix.SizeOfExcludingThisEvenIfShared(mallocSizeOf) +
         mCachedURLs.ShallowSizeOfExcludingThis(mallocSizeOf));
}

Result<FileLocation, nsresult> URLPreloader::CacheKey::ToFileLocation() {
 if (mType == TypeFile) {
   nsCOMPtr<nsIFile> file;
   MOZ_TRY(
       NS_NewLocalFile(NS_ConvertUTF8toUTF16(mPath), getter_AddRefs(file)));
   return FileLocation(file);
 }

 RefPtr<nsZipArchive> zip = Archive();
 return FileLocation(zip, mPath);
}

Result<nsCString, nsresult> URLPreloader::URLEntry::Read() {
 FileLocation location = MOZ_TRY(ToFileLocation());

 mData = MOZ_TRY(ReadLocation(location));
 return mData;
}

/* static */ Result<nsCString, nsresult> URLPreloader::URLEntry::ReadLocation(
   FileLocation& location) {
 FileLocation::Data data;
 MOZ_TRY(location.GetData(data));

 uint32_t size;
 MOZ_TRY(data.GetSize(&size));

 nsCString result;
 result.SetLength(size);
 MOZ_TRY(data.Copy(result.BeginWriting(), size));

 return std::move(result);
}

Result<nsCString, nsresult> URLPreloader::URLEntry::ReadOrWait(
   ReadType readType) {
 auto now = TimeStamp::Now();
 LOG(Info, "Reading %s\n", mPath.get());
 auto cleanup = MakeScopeExit([&]() {
   LOG(Info, "Read in %fms\n", (TimeStamp::Now() - now).ToMilliseconds());
 });

 if (mResultCode == NS_ERROR_NOT_INITIALIZED) {
   MonitorAutoLock mal(GetSingleton().mMonitor);

   while (mResultCode == NS_ERROR_NOT_INITIALIZED) {
     mal.Wait();
   }
 }

 if (mResultCode == NS_OK && mData.IsVoid()) {
   LOG(Info, "Reading synchronously...\n");
   return Read();
 }

 if (NS_FAILED(mResultCode)) {
   return Err(mResultCode);
 }

 nsCString res = mData;

 if (readType == Forget) {
   mData.SetIsVoid(true);
 }
 return res;
}

inline URLPreloader::CacheKey::CacheKey(InputBuffer& buffer) {
 Code(buffer);
 MOZ_DIAGNOSTIC_ASSERT(
     mType == TypeAppJar || mType == TypeGREJar || mType == TypeFile,
     "mType should be valid");
}

NS_IMPL_ISUPPORTS(URLPreloader, nsIMemoryReporter)

#undef LOG

}  // namespace mozilla