Bug 1286798 - Part 35: Implement database shadowing; r=asuth,janv
authorJan Varga <jan.varga@gmail.com>
Thu, 29 Nov 2018 21:49:07 +0100
changeset 508033 6214aafc061f09377c40eca490cc12621169c892
parent 508032 3adfef6668aae569ca9086120be540308c005a70
child 508034 d5f866efde44d0e6d375dc7bcb24b86f21023103
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersasuth, janv
bugs1286798
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1286798 - Part 35: Implement database shadowing; r=asuth,janv This adds synchronization to the global database used by previous local storage implementation. This patch was enhanced by asuth to bind attached database path. Places has shown that argument binding is necessary. (Profiles may include usernames in their path which can have cool emoji and such.)
dom/localstorage/ActorsParent.cpp
dom/localstorage/test/unit/test_databaseShadowing1.js
dom/localstorage/test/unit/test_databaseShadowing2.js
dom/localstorage/test/unit/xpcshell.ini
dom/storage/StorageDBThread.cpp
dom/storage/StorageUtils.cpp
dom/storage/StorageUtils.h
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -118,16 +118,22 @@ const char kPrivateBrowsingObserverTopic
 const uint32_t kDefaultOriginLimitKB = 5 * 1024;
 const uint32_t kDefaultSnapshotPrefill = 4096;
 const char kDefaultQuotaPref[] = "dom.storage.default_quota";
 const char kSnapshotPrefillPref[] = "dom.storage.snapshot_prefill";
 
 const uint32_t kPreparedDatastoreTimeoutMs = 20000;
 
 #define LS_ARCHIVE_FILE_NAME "ls-archive.sqlite"
+#define WEB_APPS_STORE_FILE_NAME "webappsstore.sqlite"
+
+// Shadow database Write Ahead Log's maximum size is 512KB
+const uint32_t kShadowMaxWALSize = 512 * 1024;
+
+const uint32_t kShadowJournalSizeLimit = kShadowMaxWALSize * 3;
 
 bool
 IsOnConnectionThread();
 
 void
 AssertIsOnConnectionThread();
 
 /*******************************************************************************
@@ -619,16 +625,270 @@ DetachArchiveDatabase(mozIStorageConnect
   ));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   return NS_OK;
 }
 
+nsresult
+GetShadowFile(const nsAString& aBasePath,
+              nsIFile** aArchiveFile)
+{
+  MOZ_ASSERT(IsOnIOThread() || IsOnConnectionThread());
+  MOZ_ASSERT(!aBasePath.IsEmpty());
+  MOZ_ASSERT(aArchiveFile);
+
+  nsCOMPtr<nsIFile> archiveFile;
+  nsresult rv = NS_NewLocalFile(aBasePath,
+                                false,
+                                getter_AddRefs(archiveFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = archiveFile->Append(NS_LITERAL_STRING(WEB_APPS_STORE_FILE_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  archiveFile.forget(aArchiveFile);
+  return NS_OK;
+}
+
+nsresult
+SetShadowJournalMode(mozIStorageConnection* aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aConnection);
+
+  // Try enabling WAL mode. This can fail in various circumstances so we have to
+  // check the results here.
+  NS_NAMED_LITERAL_CSTRING(journalModeQueryStart, "PRAGMA journal_mode = ");
+  NS_NAMED_LITERAL_CSTRING(journalModeWAL, "wal");
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  nsresult rv =
+    aConnection->CreateStatement(journalModeQueryStart + journalModeWAL,
+                                 getter_AddRefs(stmt));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool hasResult;
+  rv = stmt->ExecuteStep(&hasResult);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(hasResult);
+
+  nsCString journalMode;
+  rv = stmt->GetUTF8String(0, journalMode);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (journalMode.Equals(journalModeWAL)) {
+    // WAL mode successfully enabled. Set limits on its size here.
+
+    // Set the threshold for auto-checkpointing the WAL. We don't want giant
+    // logs slowing down us.
+    rv = aConnection->CreateStatement(NS_LITERAL_CSTRING(
+      "PRAGMA page_size;"
+    ), getter_AddRefs(stmt));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    bool hasResult;
+    rv = stmt->ExecuteStep(&hasResult);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    MOZ_ASSERT(hasResult);
+
+    int32_t pageSize;
+    rv = stmt->GetInt32(0, &pageSize);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    MOZ_ASSERT(pageSize >= 512 && pageSize <= 65536);
+
+    nsAutoCString pageCount;
+    pageCount.AppendInt(static_cast<int32_t>(kShadowMaxWALSize / pageSize));
+
+    rv = aConnection->ExecuteSimpleSQL(
+      NS_LITERAL_CSTRING("PRAGMA wal_autocheckpoint = ") + pageCount);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    // Set the maximum WAL log size to reduce footprint on mobile (large empty
+    // WAL files will be truncated)
+    nsAutoCString sizeLimit;
+    sizeLimit.AppendInt(kShadowJournalSizeLimit);
+
+    rv = aConnection->ExecuteSimpleSQL(
+      NS_LITERAL_CSTRING("PRAGMA journal_size_limit = ") + sizeLimit);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  } else {
+    rv = aConnection->ExecuteSimpleSQL(journalModeQueryStart +
+                                       NS_LITERAL_CSTRING("truncate"));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  return NS_OK;
+}
+
+nsresult
+CreateShadowStorageConnection(const nsAString& aBasePath,
+                              mozIStorageConnection** aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(!aBasePath.IsEmpty());
+  MOZ_ASSERT(aConnection);
+
+  nsCOMPtr<nsIFile> shadowFile;
+  nsresult rv = GetShadowFile(aBasePath, getter_AddRefs(shadowFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageService> ss =
+    do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = ss->OpenUnsharedDatabase(shadowFile, getter_AddRefs(connection));
+  if (rv == NS_ERROR_FILE_CORRUPTED) {
+    rv = shadowFile->Remove(false);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = ss->OpenUnsharedDatabase(shadowFile, getter_AddRefs(connection));
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = SetShadowJournalMode(connection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = StorageDBUpdater::Update(connection);
+  if (NS_FAILED(rv)) {
+    rv = connection->Close();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = shadowFile->Remove(false);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = ss->OpenUnsharedDatabase(shadowFile, getter_AddRefs(connection));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = SetShadowJournalMode(connection);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = StorageDBUpdater::Update(connection);
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  connection.forget(aConnection);
+  return NS_OK;
+}
+
+nsresult
+AttachShadowDatabase(const nsAString& aBasePath,
+                     mozIStorageConnection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(!aBasePath.IsEmpty());
+  MOZ_ASSERT(aConnection);
+
+  nsCOMPtr<nsIFile> shadowFile;
+  nsresult rv = GetShadowFile(aBasePath, getter_AddRefs(shadowFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+#ifdef DEBUG
+  bool exists;
+  rv = shadowFile->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(exists);
+#endif
+
+  nsString path;
+  rv = shadowFile->GetPath(path);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  rv = aConnection->CreateStatement(
+    NS_LITERAL_CSTRING("ATTACH DATABASE :path AS shadow;"),
+    getter_AddRefs(stmt));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("path"), path);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+DetachShadowDatabase(mozIStorageConnection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(aConnection);
+
+  nsresult rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "DETACH DATABASE shadow"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 /*******************************************************************************
  * Non-actor class declarations
  ******************************************************************************/
 
 class DatastoreOperationBase
   : public Runnable
 {
   nsCOMPtr<nsIEventTarget> mOwningEventTarget;
@@ -782,16 +1042,17 @@ private:
   class SetItemInfo;
   class RemoveItemInfo;
   class ClearInfo;
   class EndUpdateBatchOp;
   class CloseOp;
 
   RefPtr<ConnectionThread> mConnectionThread;
   nsCOMPtr<mozIStorageConnection> mStorageConnection;
+  nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
   nsTArray<nsAutoPtr<WriteInfo>> mWriteInfos;
   nsInterfaceHashtable<nsCStringHashKey, mozIStorageStatement>
     mCachedStatements;
   const nsCString mOrigin;
   const nsString mFilePath;
 #ifdef DEBUG
   bool mInUpdateBatch;
 #endif
@@ -800,16 +1061,22 @@ public:
   NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Connection)
 
   void
   AssertIsOnOwningThread() const
   {
     NS_ASSERT_OWNINGTHREAD(Connection);
   }
 
+  ArchivedOriginInfo*
+  GetArchivedOriginInfo() const
+  {
+    return mArchivedOriginInfo;
+  }
+
   // Methods which can only be called on the owning thread.
 
   // This method is used to asynchronously execute a connection datastore
   // operation on the connection thread.
   void
   Dispatch(ConnectionDatastoreOperationBase* aOp);
 
   // This method is used to asynchronously close the storage connection on the
@@ -853,32 +1120,35 @@ public:
   nsresult
   GetCachedStatement(const nsACString& aQuery,
                      CachedStatement* aCachedStatement);
 
 private:
   // Only created by ConnectionThread.
   Connection(ConnectionThread* aConnectionThread,
              const nsACString& aOrigin,
-             const nsAString& aFilePath);
+             const nsAString& aFilePath,
+             nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo);
 
   ~Connection();
 };
 
 class Connection::CachedStatement final
 {
   friend class Connection;
 
   nsCOMPtr<mozIStorageStatement> mStatement;
   Maybe<mozStorageStatementScoper> mScoper;
 
 public:
   CachedStatement();
   ~CachedStatement();
 
+  operator mozIStorageStatement*() const;
+
   mozIStorageStatement*
   operator->() const MOZ_NO_ADDREF_RELEASE_ON_RETURN;
 
 private:
   // Only called by Connection.
   void
   Assign(Connection* aConnection,
          already_AddRefed<mozIStorageStatement> aStatement);
@@ -999,17 +1269,18 @@ public:
   bool
   IsOnConnectionThread();
 
   void
   AssertIsOnConnectionThread();
 
   already_AddRefed<Connection>
   CreateConnection(const nsACString& aOrigin,
-                   const nsAString& aFilePath);
+                   const nsAString& aFilePath,
+                   nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo);
 
   void
   Shutdown();
 
   NS_INLINE_DECL_REFCOUNTING(ConnectionThread)
 
 private:
   ~ConnectionThread();
@@ -2140,16 +2411,19 @@ typedef nsDataHashtable<nsCStringHashKey
 
 // Can only be touched on the Quota Manager I/O thread.
 StaticAutoPtr<UsageHashtable> gUsages;
 
 typedef nsTHashtable<nsCStringHashKey> ArchivedOriginHashtable;
 
 StaticAutoPtr<ArchivedOriginHashtable> gArchivedOrigins;
 
+// Can only be touched on the Quota Manager I/O thread.
+bool gInitializedShadowStorage = false;
+
 bool
 IsOnConnectionThread()
 {
   MOZ_ASSERT(gConnectionThread);
   return gConnectionThread->IsOnConnectionThread();
 }
 
 void
@@ -2718,18 +2992,20 @@ ConnectionDatastoreOperationBase::Run()
 }
 
 /*******************************************************************************
  * Connection implementation
  ******************************************************************************/
 
 Connection::Connection(ConnectionThread* aConnectionThread,
                        const nsACString& aOrigin,
-                       const nsAString& aFilePath)
+                       const nsAString& aFilePath,
+                       nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo)
   : mConnectionThread(aConnectionThread)
+  , mArchivedOriginInfo(std::move(aArchivedOriginInfo))
   , mOrigin(aOrigin)
   , mFilePath(aFilePath)
 #ifdef DEBUG
   , mInUpdateBatch(false)
 #endif
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(!aOrigin.IsEmpty());
@@ -2903,16 +3179,24 @@ CachedStatement::CachedStatement()
 Connection::
 CachedStatement::~CachedStatement()
 {
   AssertIsOnConnectionThread();
 
   MOZ_COUNT_DTOR(Connection::CachedStatement);
 }
 
+Connection::
+CachedStatement::operator mozIStorageStatement*() const
+{
+  AssertIsOnConnectionThread();
+
+  return mStatement;
+}
+
 mozIStorageStatement*
 Connection::
 CachedStatement::operator->() const
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(mStatement);
 
   return mStatement;
@@ -2960,16 +3244,56 @@ SetItemInfo::Perform(Connection* aConnec
     return rv;
   }
 
   rv = stmt->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
+  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "INSERT OR REPLACE INTO shadow.webappsstore2 "
+      "(originAttributes, originKey, scope, key, value) "
+      "VALUES (:originAttributes, :originKey, :scope, :key, :value) "),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  ArchivedOriginInfo* archivedOriginInfo = aConnection->GetArchivedOriginInfo();
+
+  rv = archivedOriginInfo->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCString scope = Scheme0Scope(archivedOriginInfo->OriginSuffix(),
+                                 archivedOriginInfo->OriginNoSuffix());
+
+  rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
+                                  scope);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   return NS_OK;
 }
 
 nsresult
 Connection::
 RemoveItemInfo::Perform(Connection* aConnection)
 {
   AssertIsOnConnectionThread();
@@ -2989,16 +3313,41 @@ RemoveItemInfo::Perform(Connection* aCon
     return rv;
   }
 
   rv = stmt->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
+  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM shadow.webappsstore2 "
+      "WHERE originAttributes = :originAttributes "
+      "AND originKey = :originKey "
+      "AND key = :key;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   return NS_OK;
 }
 
 nsresult
 Connection::
 ClearInfo::Perform(Connection* aConnection)
 {
   AssertIsOnConnectionThread();
@@ -3012,30 +3361,61 @@ ClearInfo::Perform(Connection* aConnecti
     return rv;
   }
 
   rv = stmt->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
+  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM shadow.webappsstore2 "
+      "WHERE originAttributes = :originAttributes "
+      "AND originKey = :originKey;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   return NS_OK;
 }
 
 nsresult
 Connection::
 EndUpdateBatchOp::DoDatastoreWork()
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(mConnection);
 
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  nsCOMPtr<mozIStorageConnection> storageConnection =
+    mConnection->StorageConnection();
+  MOZ_ASSERT(storageConnection);
+
+  nsresult rv = AttachShadowDatabase(quotaManager->GetBasePath(),
+                                     storageConnection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   CachedStatement stmt;
-  nsresult rv =
-    mConnection->GetCachedStatement(NS_LITERAL_CSTRING("BEGIN IMMEDIATE;"),
-                                    &stmt);
+  rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING("BEGIN IMMEDIATE;"),
+                                       &stmt);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   rv = stmt->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -3049,16 +3429,21 @@ EndUpdateBatchOp::DoDatastoreWork()
     return rv;
   }
 
   rv = stmt->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
+  rv = DetachShadowDatabase(storageConnection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   return NS_OK;
 }
 
 nsresult
 Connection::
 CloseOp::DoDatastoreWork()
 {
   AssertIsOnConnectionThread();
@@ -3115,24 +3500,27 @@ ConnectionThread::IsOnConnectionThread()
 
 void
 ConnectionThread::AssertIsOnConnectionThread()
 {
   MOZ_ASSERT(IsOnConnectionThread());
 }
 
 already_AddRefed<Connection>
-ConnectionThread::CreateConnection(const nsACString& aOrigin,
-                                   const nsAString& aFilePath)
+ConnectionThread::CreateConnection(
+                            const nsACString& aOrigin,
+                            const nsAString& aFilePath,
+                            nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(!aOrigin.IsEmpty());
   MOZ_ASSERT(!mConnections.GetWeak(aOrigin));
 
-  RefPtr<Connection> connection = new Connection(this, aOrigin, aFilePath);
+  RefPtr<Connection> connection =
+    new Connection(this, aOrigin, aFilePath, std::move(aArchivedOriginInfo));
   mConnections.Put(aOrigin, connection);
 
   return connection.forget();
 }
 
 void
 ConnectionThread::Shutdown()
 {
@@ -5129,20 +5517,35 @@ PrepareDatastoreOp::DatabaseWork()
 
     mUsage = newUsage;
 
     MOZ_ASSERT(gUsages);
     MOZ_ASSERT(gUsages->Contains(mOrigin));
     gUsages->Put(mOrigin, newUsage);
   }
 
-  // Must close connection before dispatching otherwise we might race with the
-  // connection thread which needs to open the same database.
+  nsCOMPtr<mozIStorageConnection> shadowConnection;
+  if (!gInitializedShadowStorage) {
+    rv = CreateShadowStorageConnection(quotaManager->GetBasePath(),
+                                       getter_AddRefs(shadowConnection));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    gInitializedShadowStorage = true;
+  }
+
+  // Must close connections before dispatching otherwise we might race with the
+  // connection thread which needs to open the same databases.
   MOZ_ALWAYS_SUCCEEDS(connection->Close());
 
+  if (shadowConnection) {
+    MOZ_ALWAYS_SUCCEEDS(shadowConnection->Close());
+  }
+
   // Must set this before dispatching otherwise we will race with the owning
   // thread.
   mNestedState = NestedState::BeginLoadData;
 
   rv = OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -5280,18 +5683,20 @@ PrepareDatastoreOp::BeginLoadData()
       !MayProceed()) {
     return NS_ERROR_FAILURE;
   }
 
   if (!gConnectionThread) {
     gConnectionThread = new ConnectionThread();
   }
 
-  mConnection = gConnectionThread->CreateConnection(mOrigin,
-                                                    mDatabaseFilePath);
+  mConnection =
+    gConnectionThread->CreateConnection(mOrigin,
+                                        mDatabaseFilePath,
+                                        std::move(mArchivedOriginInfo));
   MOZ_ASSERT(mConnection);
 
   // Must set this before dispatching otherwise we will race with the
   // connection thread.
   mNestedState = NestedState::DatabaseWorkLoadData;
 
   // Can't assign to mLoadDataOp directly since that's a weak reference and
   // LoadDataOp is reference counted.
@@ -5949,17 +6354,17 @@ ArchivedOriginInfo::Create(nsIPrincipal*
   }
 
   return new ArchivedOriginInfo(originAttrSuffix, originKey);
 }
 
 nsresult
 ArchivedOriginInfo::BindToStatement(mozIStorageStatement* aStatement) const
 {
-  AssertIsOnIOThread();
+  MOZ_ASSERT(IsOnIOThread() || IsOnConnectionThread());
   MOZ_ASSERT(aStatement);
 
   nsresult rv =
     aStatement->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
                                      mOriginNoSuffix);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
new file mode 100644
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing1.js
@@ -0,0 +1,54 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+function* testSteps()
+{
+  const url = "http://example.com";
+
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.next_gen", true);
+
+  let principal = getPrincipal(url);
+
+  info("Getting storage");
+
+  let storage = getLocalStorage(principal);
+
+  info("Adding data");
+
+  storage.setItem("key0", "value0");
+  storage.clear();
+  storage.setItem("key1", "value1");
+  storage.removeItem("key1");
+  storage.setItem("key2", "value2");
+
+  info("Closing storage");
+
+  storage.close();
+
+  resetOrigin(principal, continueToNextStepSync);
+  yield undefined;
+
+  info("Verifying shadow database");
+
+  let profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  let shadowDatabase = profileDir.clone();
+  shadowDatabase.append("webappsstore.sqlite");
+
+  let exists = shadowDatabase.exists();
+  ok(exists, "Shadow database does exist");
+
+  info("Copying shadow database");
+
+  let currentDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile);
+  shadowDatabase.copyTo(currentDir, "");
+
+  // The shadow database is now prepared for test_databaseShadowing2.js
+
+  finishTest();
+}
new file mode 100644
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing2.js
@@ -0,0 +1,46 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+function* testSteps()
+{
+  const url = "http://example.com";
+
+  // The shadow database is prepared in test_databaseShadowing1.js
+
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.next_gen", false);
+
+  info("Verifying shadow database");
+
+  let currentDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile);
+  let shadowDatabase = currentDir.clone();
+  shadowDatabase.append("webappsstore.sqlite");
+
+  let exists = shadowDatabase.exists();
+  if (!exists) {
+    finishTest();
+    return;
+  }
+
+  info("Copying shadow database");
+
+  let profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  shadowDatabase.copyTo(profileDir, "");
+
+  info("Getting storage");
+
+  let storage = getLocalStorage(getPrincipal(url));
+
+  info("Verifying data");
+
+  ok(storage.getItem("key0") == null, "Correct value");
+  ok(storage.getItem("key1") == null, "Correct value");
+  ok(storage.getItem("key2") == "value2", "Correct value");
+
+  finishTest();
+}
--- a/dom/localstorage/test/unit/xpcshell.ini
+++ b/dom/localstorage/test/unit/xpcshell.ini
@@ -4,12 +4,16 @@
 
 [DEFAULT]
 head = head.js
 support-files =
   archive_profile.zip
   migration_profile.zip
 
 [test_archive.js]
+[test_databaseShadowing1.js]
+run-sequentially = test_databaseShadowing2.js depends on a file produced by this test
+[test_databaseShadowing2.js]
+run-sequentially = this test depends on a file produced by test_databaseShadowing1.js
 [test_eviction.js]
 [test_groupLimit.js]
 [test_migration.js]
 [test_snapshotting.js]
--- a/dom/storage/StorageDBThread.cpp
+++ b/dom/storage/StorageDBThread.cpp
@@ -53,73 +53,16 @@ using namespace StorageUtils;
 
 namespace { // anon
 
 StorageDBThread* sStorageThread = nullptr;
 
 // False until we shut the storage thread down.
 bool sStorageThreadDown = false;
 
-// This is only a compatibility code for schema version 0.  Returns the 'scope'
-// key in the schema version 0 format for the scope column.
-nsCString
-Scheme0Scope(LocalStorageCacheBridge* aCache)
-{
-  nsCString result;
-
-  nsCString suffix = aCache->OriginSuffix();
-
-  OriginAttributes oa;
-  if (!suffix.IsEmpty()) {
-    DebugOnly<bool> success = oa.PopulateFromSuffix(suffix);
-    MOZ_ASSERT(success);
-  }
-
-  if (oa.mAppId != nsIScriptSecurityManager::NO_APP_ID ||
-      oa.mInIsolatedMozBrowser) {
-    result.AppendInt(oa.mAppId);
-    result.Append(':');
-    result.Append(oa.mInIsolatedMozBrowser ? 't' : 'f');
-    result.Append(':');
-  }
-
-  // If there is more than just appid and/or inbrowser stored in origin
-  // attributes, put it to the schema 0 scope as well.  We must do that
-  // to keep the scope column unique (same resolution as schema 1 has
-  // with originAttributes and originKey columns) so that switch between
-  // schema 1 and 0 always works in both ways.
-  nsAutoCString remaining;
-  oa.mAppId = 0;
-  oa.mInIsolatedMozBrowser = false;
-  oa.CreateSuffix(remaining);
-  if (!remaining.IsEmpty()) {
-    MOZ_ASSERT(!suffix.IsEmpty());
-
-    if (result.IsEmpty()) {
-      // Must contain the old prefix, otherwise we won't search for the whole
-      // origin attributes suffix.
-      result.AppendLiteral("0:f:");
-    }
-
-    // Append the whole origin attributes suffix despite we have already stored
-    // appid and inbrowser.  We are only looking for it when the scope string
-    // starts with "$appid:$inbrowser:" (with whatever valid values).
-    //
-    // The OriginAttributes suffix is a string in a form like:
-    // "^addonId=101&userContextId=5" and it's ensured it always starts with '^'
-    // and never contains ':'.  See OriginAttributes::CreateSuffix.
-    result.Append(suffix);
-    result.Append(':');
-  }
-
-  result.Append(aCache->OriginNoSuffix());
-
-  return result;
-}
-
 } // anon
 
 // XXX Fix me!
 #if 0
 StorageDBBridge::StorageDBBridge()
 {
 }
 #endif
@@ -1151,17 +1094,18 @@ StorageDBThread::DBOperation::Perform(St
     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originAttributes"),
                                     mCache->OriginSuffix());
     NS_ENSURE_SUCCESS(rv, rv);
     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
                                     mCache->OriginNoSuffix());
     NS_ENSURE_SUCCESS(rv, rv);
     // Filling the 'scope' column just for downgrade compatibility reasons
     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
-                                    Scheme0Scope(mCache));
+                                    Scheme0Scope(mCache->OriginSuffix(),
+                                                 mCache->OriginNoSuffix()));
     NS_ENSURE_SUCCESS(rv, rv);
     rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"),
                                 mKey);
     NS_ENSURE_SUCCESS(rv, rv);
     rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"),
                                 mValue);
     NS_ENSURE_SUCCESS(rv, rv);
 
--- a/dom/storage/StorageUtils.cpp
+++ b/dom/storage/StorageUtils.cpp
@@ -108,11 +108,67 @@ CreateReversedDomain(const nsACString& a
   }
 
   ReverseString(aAsciiDomain, aKey);
 
   aKey.Append('.');
   return NS_OK;
 }
 
+// This is only a compatibility code for schema version 0.  Returns the 'scope'
+// key in the schema version 0 format for the scope column.
+nsCString
+Scheme0Scope(const nsACString& aOriginSuffix,
+             const nsACString& aOriginNoSuffix)
+{
+  nsCString result;
+
+  OriginAttributes oa;
+  if (!aOriginSuffix.IsEmpty()) {
+    DebugOnly<bool> success = oa.PopulateFromSuffix(aOriginSuffix);
+    MOZ_ASSERT(success);
+  }
+
+  if (oa.mAppId != nsIScriptSecurityManager::NO_APP_ID ||
+      oa.mInIsolatedMozBrowser) {
+    result.AppendInt(oa.mAppId);
+    result.Append(':');
+    result.Append(oa.mInIsolatedMozBrowser ? 't' : 'f');
+    result.Append(':');
+  }
+
+  // If there is more than just appid and/or inbrowser stored in origin
+  // attributes, put it to the schema 0 scope as well.  We must do that
+  // to keep the scope column unique (same resolution as schema 1 has
+  // with originAttributes and originKey columns) so that switch between
+  // schema 1 and 0 always works in both ways.
+  nsAutoCString remaining;
+  oa.mAppId = 0;
+  oa.mInIsolatedMozBrowser = false;
+  oa.CreateSuffix(remaining);
+  if (!remaining.IsEmpty()) {
+    MOZ_ASSERT(!aOriginSuffix.IsEmpty());
+
+    if (result.IsEmpty()) {
+      // Must contain the old prefix, otherwise we won't search for the whole
+      // origin attributes suffix.
+      result.AppendLiteral("0:f:");
+    }
+
+    // Append the whole origin attributes suffix despite we have already stored
+    // appid and inbrowser.  We are only looking for it when the scope string
+    // starts with "$appid:$inbrowser:" (with whatever valid values).
+    //
+    // The OriginAttributes suffix is a string in a form like:
+    // "^addonId=101&userContextId=5" and it's ensured it always starts with '^'
+    // and never contains ':'.  See OriginAttributes::CreateSuffix.
+    result.Append(aOriginSuffix);
+    result.Append(':');
+  }
+
+  result.Append(aOriginNoSuffix);
+
+  return result;
+}
+
 } // StorageUtils namespace
 } // dom namespace
 } // mozilla namespace
--- a/dom/storage/StorageUtils.h
+++ b/dom/storage/StorageUtils.h
@@ -23,13 +23,17 @@ PrincipalsEqual(nsIPrincipal* aObjectPri
 
 void
 ReverseString(const nsACString& aSource, nsACString& aResult);
 
 nsresult
 CreateReversedDomain(const nsACString& aAsciiDomain,
                      nsACString& aKey);
 
+nsCString
+Scheme0Scope(const nsACString& aOriginSuffix,
+             const nsACString& aOriginNoSuffix);
+
 } // StorageUtils namespace
 } // dom namespace
 } // mozilla namespace
 
 #endif // mozilla_dom_StorageUtils_h