[ tor-browser ].git.dasho

StorageDBThread.cpp (50152B)
      1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
      2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
      3 /* This Source Code Form is subject to the terms of the Mozilla Public
      4 * License, v. 2.0. If a copy of the MPL was not distributed with this
      5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
      6 
      7 #include "StorageDBThread.h"
      8 
      9 #include "GeckoProfiler.h"
     10 #include "LocalStorageCache.h"
     11 #include "LocalStorageManager.h"
     12 #include "StorageCommon.h"
     13 #include "StorageDBUpdater.h"
     14 #include "StorageUtils.h"
     15 #include "mozIStorageBindingParams.h"
     16 #include "mozIStorageFunction.h"
     17 #include "mozIStorageService.h"
     18 #include "mozIStorageValueArray.h"
     19 #include "mozStorageCID.h"
     20 #include "mozStorageHelper.h"
     21 #include "mozilla/BasePrincipal.h"
     22 #include "mozilla/EventQueue.h"
     23 #include "mozilla/IOInterposer.h"
     24 #include "mozilla/OriginAttributes.h"
     25 #include "mozilla/Services.h"
     26 #include "mozilla/ThreadEventQueue.h"
     27 #include "mozilla/Tokenizer.h"
     28 #include "mozilla/glean/DomStorageMetrics.h"
     29 #include "mozilla/ipc/BackgroundParent.h"
     30 #include "nsAppDirectoryServiceDefs.h"
     31 #include "nsComponentManagerUtils.h"
     32 #include "nsDirectoryServiceUtils.h"
     33 #include "nsIObserverService.h"
     34 #include "nsProxyRelease.h"
     35 #include "nsThread.h"
     36 #include "nsThreadManager.h"
     37 #include "nsThreadUtils.h"
     38 #include "nsVariant.h"
     39 
     40 // How long we collect write oprerations
     41 // before they are flushed to the database
     42 // In milliseconds.
     43 #define FLUSHING_INTERVAL_MS 5000
     44 
     45 // Write Ahead Log's maximum size is 512KB
     46 #define MAX_WAL_SIZE_BYTES 512 * 1024
     47 
     48 // Current version of the database schema
     49 #define CURRENT_SCHEMA_VERSION 2
     50 
     51 namespace mozilla::dom {
     52 
     53 using namespace StorageUtils;
     54 
     55 namespace {  // anon
     56 
     57 StorageDBThread* sStorageThread[kPrivateBrowsingIdCount] = {nullptr, nullptr};
     58 
     59 // False until we shut the storage thread down.
     60 bool sStorageThreadDown[kPrivateBrowsingIdCount] = {false, false};
     61 
     62 }  // namespace
     63 
     64 // XXX Fix me!
     65 #if 0
     66 StorageDBBridge::StorageDBBridge()
     67 {
     68 }
     69 #endif
     70 
     71 class StorageDBThread::InitHelper final : public Runnable {
     72  nsCOMPtr<nsIEventTarget> mOwningThread;
     73  mozilla::Mutex mMutex MOZ_UNANNOTATED;
     74  mozilla::CondVar mCondVar;
     75  nsString mProfilePath;
     76  nsresult mMainThreadResultCode;
     77  bool mWaiting;
     78 
     79 public:
     80  InitHelper()
     81      : Runnable("dom::StorageDBThread::InitHelper"),
     82        mOwningThread(GetCurrentSerialEventTarget()),
     83        mMutex("InitHelper::mMutex"),
     84        mCondVar(mMutex, "InitHelper::mCondVar"),
     85        mMainThreadResultCode(NS_OK),
     86        mWaiting(true) {}
     87 
     88  // Because of the `sync Preload` IPC, we need to be able to synchronously
     89  // initialize, which includes consulting and initializing
     90  // some main-thread-only APIs. Bug 1386441 discusses improving this situation.
     91  nsresult SyncDispatchAndReturnProfilePath(nsAString& aProfilePath);
     92 
     93 private:
     94  ~InitHelper() override = default;
     95 
     96  nsresult RunOnMainThread();
     97 
     98  NS_DECL_NSIRUNNABLE
     99 };
    100 
    101 class StorageDBThread::NoteBackgroundThreadRunnable final : public Runnable {
    102  // Expected to be only 0 or 1.
    103  const uint32_t mPrivateBrowsingId;
    104  nsCOMPtr<nsIEventTarget> mOwningThread;
    105 
    106 public:
    107  explicit NoteBackgroundThreadRunnable(const uint32_t aPrivateBrowsingId)
    108      : Runnable("dom::StorageDBThread::NoteBackgroundThreadRunnable"),
    109        mPrivateBrowsingId(aPrivateBrowsingId),
    110        mOwningThread(GetCurrentSerialEventTarget()) {
    111    MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
    112  }
    113 
    114 private:
    115  ~NoteBackgroundThreadRunnable() override = default;
    116 
    117  NS_DECL_NSIRUNNABLE
    118 };
    119 
    120 StorageDBThread::StorageDBThread(const uint32_t aPrivateBrowsingId)
    121    : mThread(nullptr),
    122      mThreadObserver(new ThreadObserver()),
    123      mStopIOThread(false),
    124      mWALModeEnabled(false),
    125      mDBReady(false),
    126      mStatus(NS_OK),
    127      mWorkerStatements(mWorkerConnection),
    128      mReaderStatements(mReaderConnection),
    129      mFlushImmediately(false),
    130      mPrivateBrowsingId(aPrivateBrowsingId),
    131      mPriorityCounter(0) {
    132  MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
    133 }
    134 
    135 // static
    136 StorageDBThread* StorageDBThread::Get(const uint32_t aPrivateBrowsingId) {
    137  ::mozilla::ipc::AssertIsOnBackgroundThread();
    138  MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
    139 
    140  return sStorageThread[aPrivateBrowsingId];
    141 }
    142 
    143 // static
    144 StorageDBThread* StorageDBThread::GetOrCreate(
    145    const nsString& aProfilePath, const uint32_t aPrivateBrowsingId) {
    146  ::mozilla::ipc::AssertIsOnBackgroundThread();
    147  MOZ_RELEASE_ASSERT(aPrivateBrowsingId < kPrivateBrowsingIdCount);
    148 
    149  StorageDBThread*& storageThread = sStorageThread[aPrivateBrowsingId];
    150  if (storageThread || sStorageThreadDown[aPrivateBrowsingId]) {
    151    // When sStorageThreadDown is at true, sStorageThread is null.
    152    // Checking sStorageThreadDown flag here prevents reinitialization of
    153    // the storage thread after shutdown.
    154    return storageThread;
    155  }
    156 
    157  auto newStorageThread = MakeUnique<StorageDBThread>(aPrivateBrowsingId);
    158 
    159  nsresult rv = newStorageThread->Init(aProfilePath);
    160  if (NS_WARN_IF(NS_FAILED(rv))) {
    161    return nullptr;
    162  }
    163 
    164  storageThread = newStorageThread.release();
    165 
    166  return storageThread;
    167 }
    168 
    169 // static
    170 nsresult StorageDBThread::GetProfilePath(nsString& aProfilePath) {
    171  MOZ_ASSERT(XRE_IsParentProcess());
    172  MOZ_ASSERT(NS_IsMainThread());
    173 
    174  // Need to determine location on the main thread, since
    175  // NS_GetSpecialDirectory accesses the atom table that can
    176  // only be accessed on the main thread.
    177  nsCOMPtr<nsIFile> profileDir;
    178  nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
    179                                       getter_AddRefs(profileDir));
    180  if (NS_WARN_IF(NS_FAILED(rv))) {
    181    return rv;
    182  }
    183 
    184  rv = profileDir->GetPath(aProfilePath);
    185  if (NS_WARN_IF(NS_FAILED(rv))) {
    186    return rv;
    187  }
    188 
    189  // This service has to be started on the main thread currently.
    190  nsCOMPtr<mozIStorageService> ss =
    191      do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
    192  if (NS_WARN_IF(NS_FAILED(rv))) {
    193    return rv;
    194  }
    195 
    196  return NS_OK;
    197 }
    198 
    199 nsresult StorageDBThread::Init(const nsString& aProfilePath) {
    200  ::mozilla::ipc::AssertIsOnBackgroundThread();
    201 
    202  if (mPrivateBrowsingId == 0) {
    203    nsresult rv;
    204 
    205    nsString profilePath;
    206    if (aProfilePath.IsEmpty()) {
    207      RefPtr<InitHelper> helper = new InitHelper();
    208 
    209      rv = helper->SyncDispatchAndReturnProfilePath(profilePath);
    210      if (NS_WARN_IF(NS_FAILED(rv))) {
    211        return rv;
    212      }
    213    } else {
    214      profilePath = aProfilePath;
    215    }
    216 
    217    rv = NS_NewLocalFile(profilePath, getter_AddRefs(mDatabaseFile));
    218    if (NS_WARN_IF(NS_FAILED(rv))) {
    219      return rv;
    220    }
    221 
    222    rv = mDatabaseFile->Append(u"webappsstore.sqlite"_ns);
    223    NS_ENSURE_SUCCESS(rv, rv);
    224  }
    225 
    226  // Need to keep the lock to avoid setting mThread later then
    227  // the thread body executes.
    228  MonitorAutoLock monitor(mThreadObserver->GetMonitor());
    229 
    230  mThread = PR_CreateThread(PR_USER_THREAD, &StorageDBThread::ThreadFunc, this,
    231                            PR_PRIORITY_LOW, PR_GLOBAL_THREAD,
    232                            PR_JOINABLE_THREAD, 262144);
    233  if (!mThread) {
    234    return NS_ERROR_OUT_OF_MEMORY;
    235  }
    236 
    237  RefPtr<NoteBackgroundThreadRunnable> runnable =
    238      new NoteBackgroundThreadRunnable(mPrivateBrowsingId);
    239  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(runnable));
    240 
    241  return NS_OK;
    242 }
    243 
    244 nsresult StorageDBThread::Shutdown() {
    245  ::mozilla::ipc::AssertIsOnBackgroundThread();
    246 
    247  if (!mThread) {
    248    return NS_ERROR_NOT_INITIALIZED;
    249  }
    250 
    251  auto timer = glean::localdomstorage::shutdown_database.Measure();
    252 
    253  {
    254    MonitorAutoLock monitor(mThreadObserver->GetMonitor());
    255 
    256    // After we stop, no other operations can be accepted
    257    mFlushImmediately = true;
    258    mStopIOThread = true;
    259    monitor.Notify();
    260  }
    261 
    262  PR_JoinThread(mThread);
    263  mThread = nullptr;
    264 
    265  return mStatus;
    266 }
    267 
    268 void StorageDBThread::SyncPreload(LocalStorageCacheBridge* aCache,
    269                                  bool aForceSync) {
    270  AUTO_PROFILER_LABEL("StorageDBThread::SyncPreload", OTHER);
    271  if (!aForceSync && aCache->LoadedCount()) {
    272    // Preload already started for this cache, just wait for it to finish.
    273    // LoadWait will exit after LoadDone on the cache has been called.
    274    SetHigherPriority();
    275    aCache->LoadWait();
    276    SetDefaultPriority();
    277    return;
    278  }
    279 
    280  // Bypass sync load when an update is pending in the queue to write, we would
    281  // get incosistent data in the cache.  Also don't allow sync main-thread
    282  // preload when DB open and init is still pending on the background thread.
    283  if (mDBReady && mWALModeEnabled) {
    284    bool pendingTasks;
    285    {
    286      MonitorAutoLock monitor(mThreadObserver->GetMonitor());
    287      pendingTasks = mPendingTasks.IsOriginUpdatePending(
    288                         aCache->OriginSuffix(), aCache->OriginNoSuffix()) ||
    289                     mPendingTasks.IsOriginClearPending(
    290                         aCache->OriginSuffix(), aCache->OriginNoSuffix());
    291    }
    292 
    293    if (!pendingTasks) {
    294      // WAL is enabled, thus do the load synchronously on the main thread.
    295      DBOperation preload(DBOperation::opPreload, aCache);
    296      preload.PerformAndFinalize(this);
    297      return;
    298    }
    299  }
    300 
    301  // Need to go asynchronously since WAL is not allowed or scheduled updates
    302  // need to be flushed first.
    303  // Schedule preload for this cache as the first operation.
    304  nsresult rv =
    305      InsertDBOp(MakeUnique<DBOperation>(DBOperation::opPreloadUrgent, aCache));
    306 
    307  // LoadWait exits after LoadDone of the cache has been called.
    308  if (NS_SUCCEEDED(rv)) {
    309    aCache->LoadWait();
    310  }
    311 }
    312 
    313 void StorageDBThread::AsyncFlush() {
    314  MonitorAutoLock monitor(mThreadObserver->GetMonitor());
    315  mFlushImmediately = true;
    316  monitor.Notify();
    317 }
    318 
    319 bool StorageDBThread::ShouldPreloadOrigin(const nsACString& aOrigin) {
    320  MonitorAutoLock monitor(mThreadObserver->GetMonitor());
    321  return mOriginsHavingData.Contains(aOrigin);
    322 }
    323 
    324 void StorageDBThread::GetOriginsHavingData(nsTArray<nsCString>* aOrigins) {
    325  MonitorAutoLock monitor(mThreadObserver->GetMonitor());
    326  AppendToArray(*aOrigins, mOriginsHavingData);
    327 }
    328 
    329 nsresult StorageDBThread::InsertDBOp(
    330    UniquePtr<StorageDBThread::DBOperation> aOperation) {
    331  MonitorAutoLock monitor(mThreadObserver->GetMonitor());
    332 
    333  if (NS_FAILED(mStatus)) {
    334    MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
    335    aOperation->Finalize(mStatus);
    336    return mStatus;
    337  }
    338 
    339  if (mStopIOThread) {
    340    // Thread use after shutdown demanded.
    341    MOZ_ASSERT(false);
    342    return NS_ERROR_NOT_INITIALIZED;
    343  }
    344 
    345  switch (aOperation->Type()) {
    346    case DBOperation::opPreload:
    347    case DBOperation::opPreloadUrgent:
    348      if (mPendingTasks.IsOriginUpdatePending(aOperation->OriginSuffix(),
    349                                              aOperation->OriginNoSuffix())) {
    350        // If there is a pending update operation for the scope first do the
    351        // flush before we preload the cache.  This may happen in an extremely
    352        // rare case when a child process throws away its cache before flush on
    353        // the parent has finished.  If we would preloaded the cache as a
    354        // priority operation before the pending flush, we would have got an
    355        // inconsistent cache content.
    356        mFlushImmediately = true;
    357      } else if (mPendingTasks.IsOriginClearPending(
    358                     aOperation->OriginSuffix(),
    359                     aOperation->OriginNoSuffix())) {
    360        // The scope is scheduled to be cleared, so just quickly load as empty.
    361        // We need to do this to prevent load of the DB data before the scope
    362        // has actually been cleared from the database.  Preloads are processed
    363        // immediately before update and clear operations on the database that
    364        // are flushed periodically in batches.
    365        MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
    366        aOperation->Finalize(NS_OK);
    367        return NS_OK;
    368      }
    369      [[fallthrough]];
    370 
    371    case DBOperation::opGetUsage:
    372      if (aOperation->Type() == DBOperation::opPreloadUrgent) {
    373        SetHigherPriority();  // Dropped back after urgent preload execution
    374        mPreloads.InsertElementAt(0, aOperation.release());
    375      } else {
    376        mPreloads.AppendElement(aOperation.release());
    377      }
    378 
    379      // Immediately start executing this.
    380      monitor.Notify();
    381      break;
    382 
    383    default:
    384      // Update operations are first collected, coalesced and then flushed
    385      // after a short time.
    386      mPendingTasks.Add(std::move(aOperation));
    387 
    388      ScheduleFlush();
    389      break;
    390  }
    391 
    392  return NS_OK;
    393 }
    394 
    395 void StorageDBThread::SetHigherPriority() {
    396  ++mPriorityCounter;
    397  PR_SetThreadPriority(mThread, PR_PRIORITY_URGENT);
    398 }
    399 
    400 void StorageDBThread::SetDefaultPriority() {
    401  if (--mPriorityCounter <= 0) {
    402    PR_SetThreadPriority(mThread, PR_PRIORITY_LOW);
    403  }
    404 }
    405 
    406 void StorageDBThread::ThreadFunc(void* aArg) {
    407  {
    408    auto queue = MakeRefPtr<ThreadEventQueue>(MakeUnique<EventQueue>());
    409    (void)nsThreadManager::get().CreateCurrentThread(queue);
    410  }
    411 
    412  AUTO_PROFILER_REGISTER_THREAD("localStorage DB");
    413  NS_SetCurrentThreadName("localStorage DB");
    414  mozilla::IOInterposer::RegisterCurrentThread();
    415 
    416  StorageDBThread* thread = static_cast<StorageDBThread*>(aArg);
    417  thread->ThreadFunc();
    418  mozilla::IOInterposer::UnregisterCurrentThread();
    419 }
    420 
    421 void StorageDBThread::ThreadFunc() {
    422  nsresult rv = InitDatabase();
    423 
    424  MonitorAutoLock lockMonitor(mThreadObserver->GetMonitor());
    425 
    426  if (NS_FAILED(rv)) {
    427    mStatus = rv;
    428    mStopIOThread = true;
    429    return;
    430  }
    431 
    432  // Create an nsIThread for the current PRThread, so we can observe runnables
    433  // dispatched to it.
    434  nsCOMPtr<nsIThread> thread = NS_GetCurrentThread();
    435  nsCOMPtr<nsIThreadInternal> threadInternal = do_QueryInterface(thread);
    436  MOZ_ASSERT(threadInternal);  // Should always succeed.
    437  threadInternal->SetObserver(mThreadObserver);
    438 
    439  while (MOZ_LIKELY(!mStopIOThread || mPreloads.Length() ||
    440                    mPendingTasks.HasTasks() ||
    441                    mThreadObserver->HasPendingEvents())) {
    442    // Process xpcom events first.
    443    while (MOZ_UNLIKELY(mThreadObserver->HasPendingEvents())) {
    444      mThreadObserver->ClearPendingEvents();
    445      MonitorAutoUnlock unlock(mThreadObserver->GetMonitor());
    446      bool processedEvent;
    447      do {
    448        rv = thread->ProcessNextEvent(false, &processedEvent);
    449      } while (NS_SUCCEEDED(rv) && processedEvent);
    450    }
    451 
    452    TimeDuration timeUntilFlush = TimeUntilFlush();
    453    if (MOZ_UNLIKELY(timeUntilFlush.IsZero())) {
    454      // Flush time is up or flush has been forced, do it now.
    455      UnscheduleFlush();
    456      if (mPendingTasks.Prepare()) {
    457        {
    458          MonitorAutoUnlock unlockMonitor(mThreadObserver->GetMonitor());
    459          rv = mPendingTasks.Execute(this);
    460        }
    461 
    462        if (!mPendingTasks.Finalize(rv)) {
    463          mStatus = rv;
    464          NS_WARNING("localStorage DB access broken");
    465        }
    466      }
    467      NotifyFlushCompletion();
    468    } else if (MOZ_LIKELY(mPreloads.Length())) {
    469      UniquePtr<DBOperation> op(mPreloads[0]);
    470      mPreloads.RemoveElementAt(0);
    471      {
    472        MonitorAutoUnlock unlockMonitor(mThreadObserver->GetMonitor());
    473        op->PerformAndFinalize(this);
    474      }
    475 
    476      if (op->Type() == DBOperation::opPreloadUrgent) {
    477        SetDefaultPriority();  // urgent preload unscheduled
    478      }
    479    } else if (MOZ_UNLIKELY(!mStopIOThread)) {
    480      AUTO_PROFILER_LABEL("StorageDBThread::ThreadFunc::Wait", IDLE);
    481      lockMonitor.Wait(timeUntilFlush);
    482    }
    483  }  // thread loop
    484 
    485  mStatus = ShutdownDatabase();
    486 
    487  if (threadInternal) {
    488    threadInternal->SetObserver(nullptr);
    489  }
    490 }
    491 
    492 NS_IMPL_ISUPPORTS(StorageDBThread::ThreadObserver, nsIThreadObserver)
    493 
    494 NS_IMETHODIMP
    495 StorageDBThread::ThreadObserver::OnDispatchedEvent() {
    496  MonitorAutoLock lock(mMonitor);
    497  mHasPendingEvents = true;
    498  lock.Notify();
    499  return NS_OK;
    500 }
    501 
    502 NS_IMETHODIMP
    503 StorageDBThread::ThreadObserver::OnProcessNextEvent(nsIThreadInternal* aThread,
    504                                                    bool mayWait) {
    505  return NS_OK;
    506 }
    507 
    508 NS_IMETHODIMP
    509 StorageDBThread::ThreadObserver::AfterProcessNextEvent(
    510    nsIThreadInternal* aThread, bool eventWasProcessed) {
    511  return NS_OK;
    512 }
    513 
    514 nsresult StorageDBThread::OpenDatabaseConnection() {
    515  nsresult rv;
    516 
    517  MOZ_ASSERT(!NS_IsMainThread());
    518 
    519  nsCOMPtr<mozIStorageService> service =
    520      do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
    521  NS_ENSURE_SUCCESS(rv, rv);
    522 
    523  if (mPrivateBrowsingId == 0) {
    524    MOZ_ASSERT(mDatabaseFile);
    525 
    526    rv = service->OpenUnsharedDatabase(mDatabaseFile,
    527                                       mozIStorageService::CONNECTION_DEFAULT,
    528                                       getter_AddRefs(mWorkerConnection));
    529    if (rv == NS_ERROR_FILE_CORRUPTED) {
    530      // delete the db and try opening again
    531      rv = mDatabaseFile->Remove(false);
    532      NS_ENSURE_SUCCESS(rv, rv);
    533      rv = service->OpenUnsharedDatabase(mDatabaseFile,
    534                                         mozIStorageService::CONNECTION_DEFAULT,
    535                                         getter_AddRefs(mWorkerConnection));
    536    }
    537  } else {
    538    MOZ_ASSERT(mPrivateBrowsingId == 1);
    539 
    540    rv = service->OpenSpecialDatabase(kMozStorageMemoryStorageKey,
    541                                      "lsprivatedb"_ns,
    542                                      mozIStorageService::CONNECTION_DEFAULT,
    543                                      getter_AddRefs(mWorkerConnection));
    544  }
    545  NS_ENSURE_SUCCESS(rv, rv);
    546 
    547  return NS_OK;
    548 }
    549 
    550 nsresult StorageDBThread::OpenAndUpdateDatabase() {
    551  nsresult rv;
    552 
    553  // Here we are on the worker thread. This opens the worker connection.
    554  MOZ_ASSERT(!NS_IsMainThread());
    555 
    556  rv = OpenDatabaseConnection();
    557  NS_ENSURE_SUCCESS(rv, rv);
    558 
    559  // SQLite doesn't support WAL journals for in-memory databases.
    560  if (mPrivateBrowsingId == 0) {
    561    rv = TryJournalMode();
    562    NS_ENSURE_SUCCESS(rv, rv);
    563  }
    564 
    565  return NS_OK;
    566 }
    567 
    568 nsresult StorageDBThread::InitDatabase() {
    569  nsresult rv;
    570 
    571  // Here we are on the worker thread. This opens the worker connection.
    572  MOZ_ASSERT(!NS_IsMainThread());
    573 
    574  rv = OpenAndUpdateDatabase();
    575  NS_ENSURE_SUCCESS(rv, rv);
    576 
    577  rv = StorageDBUpdater::Update(mWorkerConnection);
    578  if (NS_FAILED(rv)) {
    579    if (mPrivateBrowsingId == 0) {
    580      // Update has failed, rather throw the database away and try
    581      // opening and setting it up again.
    582      rv = mWorkerConnection->Close();
    583      mWorkerConnection = nullptr;
    584      NS_ENSURE_SUCCESS(rv, rv);
    585 
    586      rv = mDatabaseFile->Remove(false);
    587      NS_ENSURE_SUCCESS(rv, rv);
    588 
    589      rv = OpenAndUpdateDatabase();
    590    }
    591    NS_ENSURE_SUCCESS(rv, rv);
    592  }
    593 
    594  // Create a read-only clone
    595  (void)mWorkerConnection->Clone(true, getter_AddRefs(mReaderConnection));
    596  NS_ENSURE_TRUE(mReaderConnection, NS_ERROR_FAILURE);
    597 
    598  // Database open and all initiation operation are done.  Switching this flag
    599  // to true allow main thread to read directly from the database.  If we would
    600  // allow this sooner, we would have opened a window where main thread read
    601  // might operate on a totally broken and incosistent database.
    602  mDBReady = true;
    603 
    604  // List scopes having any stored data
    605  nsCOMPtr<mozIStorageStatement> stmt;
    606  // Note: result of this select must match StorageManager::CreateOrigin()
    607  rv = mWorkerConnection->CreateStatement(
    608      nsLiteralCString("SELECT DISTINCT originAttributes || ':' || originKey "
    609                       "FROM webappsstore2"),
    610      getter_AddRefs(stmt));
    611  NS_ENSURE_SUCCESS(rv, rv);
    612  mozStorageStatementScoper scope(stmt);
    613 
    614  bool exists;
    615  while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
    616    nsAutoCString foundOrigin;
    617    rv = stmt->GetUTF8String(0, foundOrigin);
    618    NS_ENSURE_SUCCESS(rv, rv);
    619 
    620    MonitorAutoLock monitor(mThreadObserver->GetMonitor());
    621    mOriginsHavingData.Insert(foundOrigin);
    622  }
    623 
    624  return NS_OK;
    625 }
    626 
    627 nsresult StorageDBThread::SetJournalMode(bool aIsWal) {
    628  nsresult rv;
    629 
    630  nsAutoCString stmtString(MOZ_STORAGE_UNIQUIFY_QUERY_STR
    631                           "PRAGMA journal_mode = ");
    632  if (aIsWal) {
    633    stmtString.AppendLiteral("wal");
    634  } else {
    635    stmtString.AppendLiteral("truncate");
    636  }
    637 
    638  nsCOMPtr<mozIStorageStatement> stmt;
    639  rv = mWorkerConnection->CreateStatement(stmtString, getter_AddRefs(stmt));
    640  NS_ENSURE_SUCCESS(rv, rv);
    641  mozStorageStatementScoper scope(stmt);
    642 
    643  bool hasResult = false;
    644  rv = stmt->ExecuteStep(&hasResult);
    645  NS_ENSURE_SUCCESS(rv, rv);
    646  if (!hasResult) {
    647    return NS_ERROR_FAILURE;
    648  }
    649 
    650  nsAutoCString journalMode;
    651  rv = stmt->GetUTF8String(0, journalMode);
    652  NS_ENSURE_SUCCESS(rv, rv);
    653  if ((aIsWal && !journalMode.EqualsLiteral("wal")) ||
    654      (!aIsWal && !journalMode.EqualsLiteral("truncate"))) {
    655    return NS_ERROR_FAILURE;
    656  }
    657 
    658  return NS_OK;
    659 }
    660 
    661 nsresult StorageDBThread::TryJournalMode() {
    662  nsresult rv;
    663 
    664  rv = SetJournalMode(true);
    665  if (NS_FAILED(rv)) {
    666    mWALModeEnabled = false;
    667 
    668    rv = SetJournalMode(false);
    669    NS_ENSURE_SUCCESS(rv, rv);
    670  } else {
    671    mWALModeEnabled = true;
    672 
    673    rv = ConfigureWALBehavior();
    674    NS_ENSURE_SUCCESS(rv, rv);
    675  }
    676 
    677  return NS_OK;
    678 }
    679 
    680 nsresult StorageDBThread::ConfigureWALBehavior() {
    681  // Get the DB's page size
    682  nsCOMPtr<mozIStorageStatement> stmt;
    683  nsresult rv = mWorkerConnection->CreateStatement(
    684      nsLiteralCString(MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"),
    685      getter_AddRefs(stmt));
    686  NS_ENSURE_SUCCESS(rv, rv);
    687 
    688  bool hasResult = false;
    689  rv = stmt->ExecuteStep(&hasResult);
    690  NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
    691 
    692  int32_t pageSize = 0;
    693  rv = stmt->GetInt32(0, &pageSize);
    694  NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && pageSize > 0, NS_ERROR_UNEXPECTED);
    695 
    696  // Set the threshold for auto-checkpointing the WAL.
    697  // We don't want giant logs slowing down reads & shutdown.
    698  // Note there is a default journal_size_limit set by mozStorage.
    699  int32_t thresholdInPages =
    700      static_cast<int32_t>(MAX_WAL_SIZE_BYTES / pageSize);
    701  nsAutoCString thresholdPragma("PRAGMA wal_autocheckpoint = ");
    702  thresholdPragma.AppendInt(thresholdInPages);
    703  rv = mWorkerConnection->ExecuteSimpleSQL(thresholdPragma);
    704  NS_ENSURE_SUCCESS(rv, rv);
    705 
    706  return NS_OK;
    707 }
    708 
    709 nsresult StorageDBThread::ShutdownDatabase() {
    710  // Has to be called on the worker thread.
    711  MOZ_ASSERT(!NS_IsMainThread());
    712 
    713  nsresult rv = mStatus;
    714 
    715  mDBReady = false;
    716 
    717  // Finalize the cached statements.
    718  mReaderStatements.FinalizeStatements();
    719  mWorkerStatements.FinalizeStatements();
    720 
    721  if (mReaderConnection) {
    722    // No need to sync access to mReaderConnection since the main thread
    723    // is right now joining this thread, unable to execute any events.
    724    mReaderConnection->Close();
    725    mReaderConnection = nullptr;
    726  }
    727 
    728  if (mWorkerConnection) {
    729    rv = mWorkerConnection->Close();
    730    mWorkerConnection = nullptr;
    731  }
    732 
    733  return rv;
    734 }
    735 
    736 void StorageDBThread::ScheduleFlush() {
    737  if (mDirtyEpoch) {
    738    return;  // Already scheduled
    739  }
    740 
    741  // Must be non-zero to indicate we are scheduled
    742  mDirtyEpoch = TimeStamp::Now();
    743 
    744  // Wake the monitor from indefinite sleep...
    745  (mThreadObserver->GetMonitor()).Notify();
    746 }
    747 
    748 void StorageDBThread::UnscheduleFlush() {
    749  // We are just about to do the flush, drop flags
    750  mFlushImmediately = false;
    751  mDirtyEpoch = TimeStamp();
    752 }
    753 
    754 TimeDuration StorageDBThread::TimeUntilFlush() {
    755  if (mFlushImmediately) {
    756    return 0;  // Do it now regardless the timeout.
    757  }
    758 
    759  if (!mDirtyEpoch) {
    760    return TimeDuration::Forever();  // No pending task...
    761  }
    762 
    763  TimeStamp now = TimeStamp::Now();
    764  TimeDuration age = now - mDirtyEpoch;
    765  static const TimeDuration kMaxAge =
    766      TimeDuration::FromMilliseconds(FLUSHING_INTERVAL_MS);
    767  if (age > kMaxAge) {
    768    return 0;  // It is time.
    769  }
    770 
    771  return kMaxAge - age;  // Time left. This is used to sleep the monitor.
    772 }
    773 
    774 void StorageDBThread::NotifyFlushCompletion() {
    775 #ifdef DOM_STORAGE_TESTS
    776  if (!NS_IsMainThread()) {
    777    RefPtr<nsRunnableMethod<StorageDBThread, void, false>> event =
    778        NewNonOwningRunnableMethod(
    779            "dom::StorageDBThread::NotifyFlushCompletion", this,
    780            &StorageDBThread::NotifyFlushCompletion);
    781    NS_DispatchToMainThread(event);
    782    return;
    783  }
    784 
    785  nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
    786  if (obs) {
    787    obs->NotifyObservers(nullptr, "domstorage-test-flushed", nullptr);
    788  }
    789 #endif
    790 }
    791 
    792 // Helper SQL function classes
    793 
    794 namespace {
    795 
    796 class OriginAttrsPatternMatchSQLFunction final : public mozIStorageFunction {
    797  NS_DECL_ISUPPORTS
    798  NS_DECL_MOZISTORAGEFUNCTION
    799 
    800  explicit OriginAttrsPatternMatchSQLFunction(
    801      OriginAttributesPattern const& aPattern)
    802      : mPattern(aPattern) {}
    803 
    804 private:
    805  OriginAttrsPatternMatchSQLFunction() = delete;
    806  ~OriginAttrsPatternMatchSQLFunction() = default;
    807 
    808  OriginAttributesPattern mPattern;
    809 };
    810 
    811 NS_IMPL_ISUPPORTS(OriginAttrsPatternMatchSQLFunction, mozIStorageFunction)
    812 
    813 NS_IMETHODIMP
    814 OriginAttrsPatternMatchSQLFunction::OnFunctionCall(
    815    mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) {
    816  nsresult rv;
    817 
    818  nsAutoCString suffix;
    819  rv = aFunctionArguments->GetUTF8String(0, suffix);
    820  NS_ENSURE_SUCCESS(rv, rv);
    821 
    822  OriginAttributes oa;
    823  bool success = oa.PopulateFromSuffix(suffix);
    824  NS_ENSURE_TRUE(success, NS_ERROR_FAILURE);
    825  bool result = mPattern.Matches(oa);
    826 
    827  RefPtr<nsVariant> outVar(new nsVariant());
    828  rv = outVar->SetAsBool(result);
    829  NS_ENSURE_SUCCESS(rv, rv);
    830 
    831  outVar.forget(aResult);
    832  return NS_OK;
    833 }
    834 
    835 }  // namespace
    836 
    837 // StorageDBThread::DBOperation
    838 
    839 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
    840                                          LocalStorageCacheBridge* aCache,
    841                                          const nsAString& aKey,
    842                                          const nsAString& aValue)
    843    : mType(aType), mCache(aCache), mKey(aKey), mValue(aValue) {
    844  MOZ_ASSERT(mType == opPreload || mType == opPreloadUrgent ||
    845             mType == opAddItem || mType == opUpdateItem ||
    846             mType == opRemoveItem || mType == opClear || mType == opClearAll);
    847  MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
    848 }
    849 
    850 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
    851                                          StorageUsageBridge* aUsage)
    852    : mType(aType), mUsage(aUsage) {
    853  MOZ_ASSERT(mType == opGetUsage);
    854  MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
    855 }
    856 
    857 StorageDBThread::DBOperation::DBOperation(const OperationType aType,
    858                                          const nsACString& aOriginNoSuffix)
    859    : mType(aType), mCache(nullptr), mOrigin(aOriginNoSuffix) {
    860  MOZ_ASSERT(mType == opClearMatchingOrigin);
    861  MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
    862 }
    863 
    864 StorageDBThread::DBOperation::DBOperation(
    865    const OperationType aType, const OriginAttributesPattern& aOriginNoSuffix)
    866    : mType(aType), mCache(nullptr), mOriginPattern(aOriginNoSuffix) {
    867  MOZ_ASSERT(mType == opClearMatchingOriginAttributes);
    868  MOZ_COUNT_CTOR(StorageDBThread::DBOperation);
    869 }
    870 
    871 StorageDBThread::DBOperation::~DBOperation() {
    872  MOZ_COUNT_DTOR(StorageDBThread::DBOperation);
    873 }
    874 
    875 const nsCString StorageDBThread::DBOperation::OriginNoSuffix() const {
    876  if (mCache) {
    877    return mCache->OriginNoSuffix();
    878  }
    879 
    880  return ""_ns;
    881 }
    882 
    883 const nsCString StorageDBThread::DBOperation::OriginSuffix() const {
    884  if (mCache) {
    885    return mCache->OriginSuffix();
    886  }
    887 
    888  return ""_ns;
    889 }
    890 
    891 const nsCString StorageDBThread::DBOperation::Origin() const {
    892  if (mCache) {
    893    return mCache->Origin();
    894  }
    895 
    896  return mOrigin;
    897 }
    898 
    899 const nsCString StorageDBThread::DBOperation::Target() const {
    900  switch (mType) {
    901    case opAddItem:
    902    case opUpdateItem:
    903    case opRemoveItem:
    904      return Origin() + "|"_ns + NS_ConvertUTF16toUTF8(mKey);
    905 
    906    default:
    907      return Origin();
    908  }
    909 }
    910 
    911 void StorageDBThread::DBOperation::PerformAndFinalize(
    912    StorageDBThread* aThread) {
    913  Finalize(Perform(aThread));
    914 }
    915 
    916 nsresult StorageDBThread::DBOperation::Perform(StorageDBThread* aThread) {
    917  nsresult rv;
    918 
    919  switch (mType) {
    920    case opPreload:
    921    case opPreloadUrgent: {
    922      // Already loaded?
    923      if (mCache->Loaded()) {
    924        break;
    925      }
    926 
    927      StatementCache* statements;
    928      if (MOZ_UNLIKELY(::mozilla::ipc::IsOnBackgroundThread())) {
    929        statements = &aThread->mReaderStatements;
    930      } else {
    931        statements = &aThread->mWorkerStatements;
    932      }
    933 
    934      // OFFSET is an optimization when we have to do a sync load
    935      // and cache has already loaded some parts asynchronously.
    936      // It skips keys we have already loaded.
    937      nsCOMPtr<mozIStorageStatement> stmt = statements->GetCachedStatement(
    938          "SELECT key, value FROM webappsstore2 "
    939          "WHERE originAttributes = :originAttributes AND originKey = "
    940          ":originKey "
    941          "ORDER BY key LIMIT -1 OFFSET :offset");
    942      NS_ENSURE_STATE(stmt);
    943      mozStorageStatementScoper scope(stmt);
    944 
    945      rv = stmt->BindUTF8StringByName("originAttributes"_ns,
    946                                      mCache->OriginSuffix());
    947      NS_ENSURE_SUCCESS(rv, rv);
    948 
    949      rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
    950      NS_ENSURE_SUCCESS(rv, rv);
    951 
    952      rv = stmt->BindInt32ByName("offset"_ns,
    953                                 static_cast<int32_t>(mCache->LoadedCount()));
    954      NS_ENSURE_SUCCESS(rv, rv);
    955 
    956      bool exists;
    957      while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
    958        nsAutoString key;
    959        rv = stmt->GetString(0, key);
    960        NS_ENSURE_SUCCESS(rv, rv);
    961 
    962        nsAutoString value;
    963        rv = stmt->GetString(1, value);
    964        NS_ENSURE_SUCCESS(rv, rv);
    965 
    966        if (!mCache->LoadItem(key, value)) {
    967          break;
    968        }
    969      }
    970      // The loop condition's call to ExecuteStep() may have terminated because
    971      // !NS_SUCCEEDED(), we need an early return to cover that case.  This also
    972      // covers success cases as well, but that's inductively safe.
    973      NS_ENSURE_SUCCESS(rv, rv);
    974      break;
    975    }
    976 
    977    case opGetUsage: {
    978      // Bug 1676410 fixed a regression caused by bug 1165214. However, it
    979      // turns out that 100% correct checking of the eTLD+1 usage is not
    980      // possible to recover easily, see bug 1683299.
    981 #if 0
    982      // This is how it should be done, but due to other problems like lack
    983      // of usage synchronization between content processes, we temporarily
    984      // disabled the matching using "%".
    985 
    986      nsCOMPtr<mozIStorageStatement> stmt =
    987          aThread->mWorkerStatements.GetCachedStatement(
    988              "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2 "
    989              "WHERE (originAttributes || ':' || originKey) LIKE :usageOrigin "
    990              "ESCAPE '\\'");
    991      NS_ENSURE_STATE(stmt);
    992 
    993      mozStorageStatementScoper scope(stmt);
    994 
    995      // The database schema is built around cleverly reversing domain names
    996      // (the "originKey") so that we can efficiently group usage by eTLD+1.
    997      // "foo.example.org" has an eTLD+1 of ".example.org". They reverse to
    998      // "gro.elpmaxe.oof" and "gro.elpmaxe." respectively, noting that the
    999      // reversed eTLD+1 is a prefix of its reversed sub-domain. To this end,
   1000      // we can calculate all of the usage for an eTLD+1 by summing up all the
   1001      // rows which have the reversed eTLD+1 as a prefix. In SQL we can
   1002      // accomplish this using LIKE which provides for case-insensitive
   1003      // matching with "_" as a single-character wildcard match and "%" any
   1004      // sequence of zero or more characters. So by suffixing the reversed
   1005      // eTLD+1 and using "%" we get our case-insensitive (domain names are
   1006      // case-insensitive) matching. Note that although legal domain names
   1007      // don't include "_" or "%", file origins can include them, so we need
   1008      // to escape our OriginScope for correctness.
   1009      nsAutoCString originScopeEscaped;
   1010      rv = stmt->EscapeUTF8StringForLIKE(mUsage->OriginScope(), '\\',
   1011                                         originScopeEscaped);
   1012      NS_ENSURE_SUCCESS(rv, rv);
   1013 
   1014      rv = stmt->BindUTF8StringByName("usageOrigin"_ns,
   1015                                      originScopeEscaped + "%"_ns);
   1016      NS_ENSURE_SUCCESS(rv, rv);
   1017 #else
   1018      // This is the code before bug 1676410 and bug 1676973. The returned
   1019      // usage will be zero in most of the cases, but due to lack of usage
   1020      // synchronization between content processes we have to live with this
   1021      // semi-broken behaviour because it causes less harm than the matching
   1022      // using "%".
   1023 
   1024      nsCOMPtr<mozIStorageStatement> stmt =
   1025          aThread->mWorkerStatements.GetCachedStatement(
   1026              "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2 "
   1027              "WHERE (originAttributes || ':' || originKey) LIKE :usageOrigin");
   1028      NS_ENSURE_STATE(stmt);
   1029 
   1030      mozStorageStatementScoper scope(stmt);
   1031 
   1032      rv = stmt->BindUTF8StringByName("usageOrigin"_ns, mUsage->OriginScope());
   1033      NS_ENSURE_SUCCESS(rv, rv);
   1034 #endif
   1035 
   1036      bool exists;
   1037      rv = stmt->ExecuteStep(&exists);
   1038      NS_ENSURE_SUCCESS(rv, rv);
   1039 
   1040      int64_t usage = 0;
   1041      if (exists) {
   1042        rv = stmt->GetInt64(0, &usage);
   1043        NS_ENSURE_SUCCESS(rv, rv);
   1044      }
   1045 
   1046      mUsage->LoadUsage(usage);
   1047      break;
   1048    }
   1049 
   1050    case opAddItem:
   1051    case opUpdateItem: {
   1052      MOZ_ASSERT(!NS_IsMainThread());
   1053 
   1054      nsCOMPtr<mozIStorageStatement> stmt =
   1055          aThread->mWorkerStatements.GetCachedStatement(
   1056              "INSERT OR REPLACE INTO webappsstore2 (originAttributes, "
   1057              "originKey, scope, key, value) "
   1058              "VALUES (:originAttributes, :originKey, :scope, :key, :value) ");
   1059      NS_ENSURE_STATE(stmt);
   1060 
   1061      mozStorageStatementScoper scope(stmt);
   1062 
   1063      rv = stmt->BindUTF8StringByName("originAttributes"_ns,
   1064                                      mCache->OriginSuffix());
   1065      NS_ENSURE_SUCCESS(rv, rv);
   1066      rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
   1067      NS_ENSURE_SUCCESS(rv, rv);
   1068      // Filling the 'scope' column just for downgrade compatibility reasons
   1069      rv = stmt->BindUTF8StringByName(
   1070          "scope"_ns,
   1071          Scheme0Scope(mCache->OriginSuffix(), mCache->OriginNoSuffix()));
   1072      NS_ENSURE_SUCCESS(rv, rv);
   1073      rv = stmt->BindStringByName("key"_ns, mKey);
   1074      NS_ENSURE_SUCCESS(rv, rv);
   1075      rv = stmt->BindStringByName("value"_ns, mValue);
   1076      NS_ENSURE_SUCCESS(rv, rv);
   1077 
   1078      rv = stmt->Execute();
   1079      NS_ENSURE_SUCCESS(rv, rv);
   1080 
   1081      MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
   1082      aThread->mOriginsHavingData.Insert(Origin());
   1083      break;
   1084    }
   1085 
   1086    case opRemoveItem: {
   1087      MOZ_ASSERT(!NS_IsMainThread());
   1088 
   1089      nsCOMPtr<mozIStorageStatement> stmt =
   1090          aThread->mWorkerStatements.GetCachedStatement(
   1091              "DELETE FROM webappsstore2 "
   1092              "WHERE originAttributes = :originAttributes AND originKey = "
   1093              ":originKey "
   1094              "AND key = :key ");
   1095      NS_ENSURE_STATE(stmt);
   1096      mozStorageStatementScoper scope(stmt);
   1097 
   1098      rv = stmt->BindUTF8StringByName("originAttributes"_ns,
   1099                                      mCache->OriginSuffix());
   1100      NS_ENSURE_SUCCESS(rv, rv);
   1101      rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
   1102      NS_ENSURE_SUCCESS(rv, rv);
   1103      rv = stmt->BindStringByName("key"_ns, mKey);
   1104      NS_ENSURE_SUCCESS(rv, rv);
   1105 
   1106      rv = stmt->Execute();
   1107      NS_ENSURE_SUCCESS(rv, rv);
   1108 
   1109      break;
   1110    }
   1111 
   1112    case opClear: {
   1113      MOZ_ASSERT(!NS_IsMainThread());
   1114 
   1115      nsCOMPtr<mozIStorageStatement> stmt =
   1116          aThread->mWorkerStatements.GetCachedStatement(
   1117              "DELETE FROM webappsstore2 "
   1118              "WHERE originAttributes = :originAttributes AND originKey = "
   1119              ":originKey");
   1120      NS_ENSURE_STATE(stmt);
   1121      mozStorageStatementScoper scope(stmt);
   1122 
   1123      rv = stmt->BindUTF8StringByName("originAttributes"_ns,
   1124                                      mCache->OriginSuffix());
   1125      NS_ENSURE_SUCCESS(rv, rv);
   1126      rv = stmt->BindUTF8StringByName("originKey"_ns, mCache->OriginNoSuffix());
   1127      NS_ENSURE_SUCCESS(rv, rv);
   1128 
   1129      rv = stmt->Execute();
   1130      NS_ENSURE_SUCCESS(rv, rv);
   1131 
   1132      MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
   1133      aThread->mOriginsHavingData.Remove(Origin());
   1134      break;
   1135    }
   1136 
   1137    case opClearAll: {
   1138      MOZ_ASSERT(!NS_IsMainThread());
   1139 
   1140      nsCOMPtr<mozIStorageStatement> stmt =
   1141          aThread->mWorkerStatements.GetCachedStatement(
   1142              "DELETE FROM webappsstore2");
   1143      NS_ENSURE_STATE(stmt);
   1144      mozStorageStatementScoper scope(stmt);
   1145 
   1146      rv = stmt->Execute();
   1147      NS_ENSURE_SUCCESS(rv, rv);
   1148 
   1149      MonitorAutoLock monitor(aThread->mThreadObserver->GetMonitor());
   1150      aThread->mOriginsHavingData.Clear();
   1151      break;
   1152    }
   1153 
   1154    case opClearMatchingOrigin: {
   1155      MOZ_ASSERT(!NS_IsMainThread());
   1156 
   1157      nsCOMPtr<mozIStorageStatement> stmt =
   1158          aThread->mWorkerStatements.GetCachedStatement(
   1159              "DELETE FROM webappsstore2"
   1160              " WHERE originKey GLOB :scope");
   1161      NS_ENSURE_STATE(stmt);
   1162      mozStorageStatementScoper scope(stmt);
   1163 
   1164      rv = stmt->BindUTF8StringByName("scope"_ns, mOrigin + "*"_ns);
   1165      NS_ENSURE_SUCCESS(rv, rv);
   1166 
   1167      rv = stmt->Execute();
   1168      NS_ENSURE_SUCCESS(rv, rv);
   1169 
   1170      // No need to selectively clear mOriginsHavingData here.  That hashtable
   1171      // only prevents preload for scopes with no data.  Leaving a false record
   1172      // in it has a negligible effect on performance.
   1173      break;
   1174    }
   1175 
   1176    case opClearMatchingOriginAttributes: {
   1177      MOZ_ASSERT(!NS_IsMainThread());
   1178 
   1179      // Register the ORIGIN_ATTRS_PATTERN_MATCH function, initialized with the
   1180      // pattern
   1181      nsCOMPtr<mozIStorageFunction> patternMatchFunction(
   1182          new OriginAttrsPatternMatchSQLFunction(mOriginPattern));
   1183 
   1184      rv = aThread->mWorkerConnection->CreateFunction(
   1185          "ORIGIN_ATTRS_PATTERN_MATCH"_ns, 1, patternMatchFunction);
   1186      NS_ENSURE_SUCCESS(rv, rv);
   1187 
   1188      nsCOMPtr<mozIStorageStatement> stmt =
   1189          aThread->mWorkerStatements.GetCachedStatement(
   1190              "DELETE FROM webappsstore2"
   1191              " WHERE ORIGIN_ATTRS_PATTERN_MATCH(originAttributes)");
   1192 
   1193      if (stmt) {
   1194        mozStorageStatementScoper scope(stmt);
   1195        rv = stmt->Execute();
   1196      } else {
   1197        rv = NS_ERROR_UNEXPECTED;
   1198      }
   1199 
   1200      // Always remove the function
   1201      aThread->mWorkerConnection->RemoveFunction(
   1202          "ORIGIN_ATTRS_PATTERN_MATCH"_ns);
   1203 
   1204      NS_ENSURE_SUCCESS(rv, rv);
   1205 
   1206      // No need to selectively clear mOriginsHavingData here.  That hashtable
   1207      // only prevents preload for scopes with no data.  Leaving a false record
   1208      // in it has a negligible effect on performance.
   1209      break;
   1210    }
   1211 
   1212    default:
   1213      NS_ERROR("Unknown task type");
   1214      break;
   1215  }
   1216 
   1217  return NS_OK;
   1218 }
   1219 
   1220 void StorageDBThread::DBOperation::Finalize(nsresult aRv) {
   1221  switch (mType) {
   1222    case opPreloadUrgent:
   1223    case opPreload:
   1224      if (NS_FAILED(aRv)) {
   1225        // When we are here, something failed when loading from the database.
   1226        // Notify that the storage is loaded to prevent deadlock of the main
   1227        // thread, even though it is actually empty or incomplete.
   1228        NS_WARNING("Failed to preload localStorage");
   1229      }
   1230 
   1231      mCache->LoadDone(aRv);
   1232      break;
   1233 
   1234    case opGetUsage:
   1235      if (NS_FAILED(aRv)) {
   1236        mUsage->LoadUsage(0);
   1237      }
   1238 
   1239      break;
   1240 
   1241    default:
   1242      if (NS_FAILED(aRv)) {
   1243        NS_WARNING(
   1244            "localStorage update/clear operation failed,"
   1245            " data may not persist or clean up");
   1246      }
   1247 
   1248      break;
   1249  }
   1250 }
   1251 
   1252 // StorageDBThread::PendingOperations
   1253 
   1254 StorageDBThread::PendingOperations::PendingOperations()
   1255    : mFlushFailureCount(0) {}
   1256 
   1257 bool StorageDBThread::PendingOperations::HasTasks() const {
   1258  return !!mUpdates.Count() || !!mClears.Count();
   1259 }
   1260 
   1261 namespace {
   1262 
   1263 bool OriginPatternMatches(const nsACString& aOriginSuffix,
   1264                          const OriginAttributesPattern& aPattern) {
   1265  OriginAttributes oa;
   1266  DebugOnly<bool> rv = oa.PopulateFromSuffix(aOriginSuffix);
   1267  MOZ_ASSERT(rv);
   1268  return aPattern.Matches(oa);
   1269 }
   1270 
   1271 }  // namespace
   1272 
   1273 bool StorageDBThread::PendingOperations::CheckForCoalesceOpportunity(
   1274    DBOperation* aNewOp, DBOperation::OperationType aPendingType,
   1275    DBOperation::OperationType aNewType) {
   1276  if (aNewOp->Type() != aNewType) {
   1277    return false;
   1278  }
   1279 
   1280  StorageDBThread::DBOperation* pendingTask;
   1281  if (!mUpdates.Get(aNewOp->Target(), &pendingTask)) {
   1282    return false;
   1283  }
   1284 
   1285  if (pendingTask->Type() != aPendingType) {
   1286    return false;
   1287  }
   1288 
   1289  return true;
   1290 }
   1291 
   1292 void StorageDBThread::PendingOperations::Add(
   1293    UniquePtr<StorageDBThread::DBOperation> aOperation) {
   1294  // Optimize: when a key to remove has never been written to disk
   1295  // just bypass this operation.  A key is new when an operation scheduled
   1296  // to write it to the database is of type opAddItem.
   1297  if (CheckForCoalesceOpportunity(aOperation.get(), DBOperation::opAddItem,
   1298                                  DBOperation::opRemoveItem)) {
   1299    mUpdates.Remove(aOperation->Target());
   1300    return;
   1301  }
   1302 
   1303  // Optimize: when changing a key that is new and has never been
   1304  // written to disk, keep type of the operation to store it at opAddItem.
   1305  // This allows optimization to just forget adding a new key when
   1306  // it is removed from the storage before flush.
   1307  if (CheckForCoalesceOpportunity(aOperation.get(), DBOperation::opAddItem,
   1308                                  DBOperation::opUpdateItem)) {
   1309    aOperation->mType = DBOperation::opAddItem;
   1310  }
   1311 
   1312  // Optimize: to prevent lose of remove operation on a key when doing
   1313  // remove/set/remove on a previously existing key we have to change
   1314  // opAddItem to opUpdateItem on the new operation when there is opRemoveItem
   1315  // pending for the key.
   1316  if (CheckForCoalesceOpportunity(aOperation.get(), DBOperation::opRemoveItem,
   1317                                  DBOperation::opAddItem)) {
   1318    aOperation->mType = DBOperation::opUpdateItem;
   1319  }
   1320 
   1321  switch (aOperation->Type()) {
   1322      // Operations on single keys
   1323 
   1324    case DBOperation::opAddItem:
   1325    case DBOperation::opUpdateItem:
   1326    case DBOperation::opRemoveItem:
   1327      // Override any existing operation for the target (=scope+key).
   1328      mUpdates.InsertOrUpdate(aOperation->Target(), std::move(aOperation));
   1329      break;
   1330 
   1331      // Clear operations
   1332 
   1333    case DBOperation::opClear:
   1334    case DBOperation::opClearMatchingOrigin:
   1335    case DBOperation::opClearMatchingOriginAttributes:
   1336      // Drop all update (insert/remove) operations for equivavelent or matching
   1337      // scope.  We do this as an optimization as well as a must based on the
   1338      // logic, if we would not delete the update tasks, changes would have been
   1339      // stored to the database after clear operations have been executed.
   1340      for (auto iter = mUpdates.Iter(); !iter.Done(); iter.Next()) {
   1341        const auto& pendingTask = iter.Data();
   1342 
   1343        if (aOperation->Type() == DBOperation::opClear &&
   1344            (pendingTask->OriginNoSuffix() != aOperation->OriginNoSuffix() ||
   1345             pendingTask->OriginSuffix() != aOperation->OriginSuffix())) {
   1346          continue;
   1347        }
   1348 
   1349        if (aOperation->Type() == DBOperation::opClearMatchingOrigin &&
   1350            !StringBeginsWith(pendingTask->OriginNoSuffix(),
   1351                              aOperation->Origin())) {
   1352          continue;
   1353        }
   1354 
   1355        if (aOperation->Type() ==
   1356                DBOperation::opClearMatchingOriginAttributes &&
   1357            !OriginPatternMatches(pendingTask->OriginSuffix(),
   1358                                  aOperation->OriginPattern())) {
   1359          continue;
   1360        }
   1361 
   1362        iter.Remove();
   1363      }
   1364 
   1365      mClears.InsertOrUpdate(aOperation->Target(), std::move(aOperation));
   1366      break;
   1367 
   1368    case DBOperation::opClearAll:
   1369      // Drop simply everything, this is a super-operation.
   1370      mUpdates.Clear();
   1371      mClears.Clear();
   1372      mClears.InsertOrUpdate(aOperation->Target(), std::move(aOperation));
   1373      break;
   1374 
   1375    default:
   1376      MOZ_ASSERT(false);
   1377      break;
   1378  }
   1379 }
   1380 
   1381 bool StorageDBThread::PendingOperations::Prepare() {
   1382  // Called under the lock
   1383 
   1384  // First collect clear operations and then updates, we can
   1385  // do this since whenever a clear operation for a scope is
   1386  // scheduled, we drop all updates matching that scope. So,
   1387  // all scope-related update operations we have here now were
   1388  // scheduled after the clear operations.
   1389  for (auto iter = mClears.Iter(); !iter.Done(); iter.Next()) {
   1390    mExecList.AppendElement(std::move(iter.Data()));
   1391  }
   1392  mClears.Clear();
   1393 
   1394  for (auto iter = mUpdates.Iter(); !iter.Done(); iter.Next()) {
   1395    mExecList.AppendElement(std::move(iter.Data()));
   1396  }
   1397  mUpdates.Clear();
   1398 
   1399  return !!mExecList.Length();
   1400 }
   1401 
   1402 nsresult StorageDBThread::PendingOperations::Execute(StorageDBThread* aThread) {
   1403  // Called outside the lock
   1404 
   1405  mozStorageTransaction transaction(aThread->mWorkerConnection, false);
   1406 
   1407  nsresult rv = transaction.Start();
   1408  if (NS_FAILED(rv)) {
   1409    return rv;
   1410  }
   1411 
   1412  for (uint32_t i = 0; i < mExecList.Length(); ++i) {
   1413    const auto& task = mExecList[i];
   1414    rv = task->Perform(aThread);
   1415    if (NS_FAILED(rv)) {
   1416      return rv;
   1417    }
   1418  }
   1419 
   1420  rv = transaction.Commit();
   1421  if (NS_FAILED(rv)) {
   1422    return rv;
   1423  }
   1424 
   1425  return NS_OK;
   1426 }
   1427 
   1428 bool StorageDBThread::PendingOperations::Finalize(nsresult aRv) {
   1429  // Called under the lock
   1430 
   1431  // The list is kept on a failure to retry it
   1432  if (NS_FAILED(aRv)) {
   1433    // XXX Followup: we may try to reopen the database and flush these
   1434    // pending tasks, however testing showed that even though I/O is actually
   1435    // broken some amount of operations is left in sqlite+system buffers and
   1436    // seems like successfully flushed to disk.
   1437    // Tested by removing a flash card and disconnecting from network while
   1438    // using a network drive on Windows system.
   1439    NS_WARNING("Flush operation on localStorage database failed");
   1440 
   1441    ++mFlushFailureCount;
   1442 
   1443    return mFlushFailureCount >= 5;
   1444  }
   1445 
   1446  mFlushFailureCount = 0;
   1447  mExecList.Clear();
   1448  return true;
   1449 }
   1450 
   1451 namespace {
   1452 
   1453 bool FindPendingClearForOrigin(
   1454    const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix,
   1455    StorageDBThread::DBOperation* aPendingOperation) {
   1456  if (aPendingOperation->Type() == StorageDBThread::DBOperation::opClearAll) {
   1457    return true;
   1458  }
   1459 
   1460  if (aPendingOperation->Type() == StorageDBThread::DBOperation::opClear &&
   1461      aOriginNoSuffix == aPendingOperation->OriginNoSuffix() &&
   1462      aOriginSuffix == aPendingOperation->OriginSuffix()) {
   1463    return true;
   1464  }
   1465 
   1466  if (aPendingOperation->Type() ==
   1467          StorageDBThread::DBOperation::opClearMatchingOrigin &&
   1468      StringBeginsWith(aOriginNoSuffix, aPendingOperation->Origin())) {
   1469    return true;
   1470  }
   1471 
   1472  if (aPendingOperation->Type() ==
   1473          StorageDBThread::DBOperation::opClearMatchingOriginAttributes &&
   1474      OriginPatternMatches(aOriginSuffix, aPendingOperation->OriginPattern())) {
   1475    return true;
   1476  }
   1477 
   1478  return false;
   1479 }
   1480 
   1481 }  // namespace
   1482 
   1483 bool StorageDBThread::PendingOperations::IsOriginClearPending(
   1484    const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix) const {
   1485  // Called under the lock
   1486 
   1487  for (const auto& clear : mClears.Values()) {
   1488    if (FindPendingClearForOrigin(aOriginSuffix, aOriginNoSuffix,
   1489                                  clear.get())) {
   1490      return true;
   1491    }
   1492  }
   1493 
   1494  for (uint32_t i = 0; i < mExecList.Length(); ++i) {
   1495    if (FindPendingClearForOrigin(aOriginSuffix, aOriginNoSuffix,
   1496                                  mExecList[i].get())) {
   1497      return true;
   1498    }
   1499  }
   1500 
   1501  return false;
   1502 }
   1503 
   1504 namespace {
   1505 
   1506 bool FindPendingUpdateForOrigin(
   1507    const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix,
   1508    StorageDBThread::DBOperation* aPendingOperation) {
   1509  if ((aPendingOperation->Type() == StorageDBThread::DBOperation::opAddItem ||
   1510       aPendingOperation->Type() ==
   1511           StorageDBThread::DBOperation::opUpdateItem ||
   1512       aPendingOperation->Type() ==
   1513           StorageDBThread::DBOperation::opRemoveItem) &&
   1514      aOriginNoSuffix == aPendingOperation->OriginNoSuffix() &&
   1515      aOriginSuffix == aPendingOperation->OriginSuffix()) {
   1516    return true;
   1517  }
   1518 
   1519  return false;
   1520 }
   1521 
   1522 }  // namespace
   1523 
   1524 bool StorageDBThread::PendingOperations::IsOriginUpdatePending(
   1525    const nsACString& aOriginSuffix, const nsACString& aOriginNoSuffix) const {
   1526  // Called under the lock
   1527 
   1528  for (const auto& update : mUpdates.Values()) {
   1529    if (FindPendingUpdateForOrigin(aOriginSuffix, aOriginNoSuffix,
   1530                                   update.get())) {
   1531      return true;
   1532    }
   1533  }
   1534 
   1535  for (uint32_t i = 0; i < mExecList.Length(); ++i) {
   1536    if (FindPendingUpdateForOrigin(aOriginSuffix, aOriginNoSuffix,
   1537                                   mExecList[i].get())) {
   1538      return true;
   1539    }
   1540  }
   1541 
   1542  return false;
   1543 }
   1544 
   1545 nsresult StorageDBThread::InitHelper::SyncDispatchAndReturnProfilePath(
   1546    nsAString& aProfilePath) {
   1547  ::mozilla::ipc::AssertIsOnBackgroundThread();
   1548 
   1549  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
   1550 
   1551  mozilla::MutexAutoLock autolock(mMutex);
   1552  while (mWaiting) {
   1553    mCondVar.Wait();
   1554  }
   1555 
   1556  if (NS_WARN_IF(NS_FAILED(mMainThreadResultCode))) {
   1557    return mMainThreadResultCode;
   1558  }
   1559 
   1560  aProfilePath = mProfilePath;
   1561  return NS_OK;
   1562 }
   1563 
   1564 NS_IMETHODIMP
   1565 StorageDBThread::InitHelper::Run() {
   1566  MOZ_ASSERT(NS_IsMainThread());
   1567 
   1568  nsresult rv = GetProfilePath(mProfilePath);
   1569  if (NS_WARN_IF(NS_FAILED(rv))) {
   1570    mMainThreadResultCode = rv;
   1571  }
   1572 
   1573  mozilla::MutexAutoLock lock(mMutex);
   1574  MOZ_ASSERT(mWaiting);
   1575 
   1576  mWaiting = false;
   1577  mCondVar.Notify();
   1578 
   1579  return NS_OK;
   1580 }
   1581 
   1582 NS_IMETHODIMP
   1583 StorageDBThread::NoteBackgroundThreadRunnable::Run() {
   1584  MOZ_ASSERT(NS_IsMainThread());
   1585 
   1586  StorageObserver* observer = StorageObserver::Self();
   1587  MOZ_ASSERT(observer);
   1588 
   1589  observer->NoteBackgroundThread(mPrivateBrowsingId, mOwningThread);
   1590 
   1591  return NS_OK;
   1592 }
   1593 
   1594 NS_IMETHODIMP
   1595 StorageDBThread::ShutdownRunnable::Run() {
   1596  if (NS_IsMainThread()) {
   1597    mDone = true;
   1598 
   1599    return NS_OK;
   1600  }
   1601 
   1602  ::mozilla::ipc::AssertIsOnBackgroundThread();
   1603  MOZ_RELEASE_ASSERT(mPrivateBrowsingId < kPrivateBrowsingIdCount);
   1604 
   1605  StorageDBThread*& storageThread = sStorageThread[mPrivateBrowsingId];
   1606  if (storageThread) {
   1607    sStorageThreadDown[mPrivateBrowsingId] = true;
   1608 
   1609    storageThread->Shutdown();
   1610 
   1611    delete storageThread;
   1612    storageThread = nullptr;
   1613  }
   1614 
   1615  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
   1616 
   1617  return NS_OK;
   1618 }
   1619 
   1620 }  // namespace mozilla::dom
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE