Bug 1546310 - LSNG: Parent process should store values in UTF8; r=asuth
☠☠ backed out by df3eadfa74a8 ☠ ☠
authorJan Varga <jan.varga@gmail.com>
Mon, 29 Apr 2019 06:09:42 +0200
changeset 531467 df97dfbb526adfe2fdf14a94dcc6bf83b5a2cab5
parent 531466 ade08d6dc3614c3ec3a3adee85a058cf666831f8
child 531468 d1bd31177b1456ec930df76ba7806806bc919891
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersasuth
bugs1546310
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1546310 - LSNG: Parent process should store values in UTF8; r=asuth Differential Revision: https://phabricator.services.mozilla.com/D29139
dom/localstorage/ActorsChild.cpp
dom/localstorage/ActorsParent.cpp
dom/localstorage/LSSnapshot.cpp
dom/localstorage/LSValue.cpp
dom/localstorage/LSValue.h
dom/localstorage/SerializationHelpers.h
dom/quota/ActorsParent.cpp
dom/quota/QuotaManager.h
dom/quota/test/unit/localStorageArchive2upgrade_profile.zip
dom/quota/test/unit/test_localStorageArchive2upgrade.js
dom/quota/test/unit/xpcshell.ini
--- a/dom/localstorage/ActorsChild.cpp
+++ b/dom/localstorage/ActorsChild.cpp
@@ -135,18 +135,18 @@ mozilla::ipc::IPCResult LSObserverChild:
 
   nsresult rv;
   nsCOMPtr<nsIPrincipal> principal =
       PrincipalInfoToPrincipal(aPrincipalInfo, &rv);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return IPC_FAIL_NO_REASON(this);
   }
 
-  Storage::NotifyChange(/* aStorage */ nullptr, principal, aKey, aOldValue,
-                        aNewValue,
+  Storage::NotifyChange(/* aStorage */ nullptr, principal, aKey,
+                        aOldValue.AsString(), aNewValue.AsString(),
                         /* aStorageType */ kLocalStorageType, aDocumentURI,
                         /* aIsPrivate */ !!aPrivateBrowsingId,
                         /* aImmediateDispatch */ true);
 
   return IPC_OK();
 }
 
 /*******************************************************************************
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -88,17 +88,17 @@ class Snapshot;
 typedef nsClassHashtable<nsCStringHashKey, ArchivedOriginInfo>
     ArchivedOriginHashtable;
 
 /*******************************************************************************
  * Constants
  ******************************************************************************/
 
 // Major schema version. Bump for almost everything.
-const uint32_t kMajorSchemaVersion = 2;
+const uint32_t kMajorSchemaVersion = 3;
 
 // Minor schema version. Should almost always be 0 (maybe bump on release
 // branches if we have to).
 const uint32_t kMinorSchemaVersion = 0;
 
 // The schema version we store in the SQLite database is a (signed) 32-bit
 // integer. The major version is left-shifted 4 bits so the max value is
 // 0xFFFFFFF. The minor version occupies the lower 4 bits and its max is 0xF.
@@ -316,16 +316,17 @@ nsresult CreateTables(mozIStorageConnect
     return rv;
   }
 
   // Table `data`
   rv = aConnection->ExecuteSimpleSQL(
       NS_LITERAL_CSTRING("CREATE TABLE data"
                          "( key TEXT PRIMARY KEY"
                          ", value TEXT NOT NULL"
+                         ", utf16Length INTEGER NOT NULL DEFAULT 0"
                          ", compressed INTEGER NOT NULL DEFAULT 0"
                          ", lastAccessTime INTEGER NOT NULL DEFAULT 0"
                          ");"));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   rv = aConnection->SetSchemaVersion(kSQLiteSchemaVersion);
@@ -357,16 +358,42 @@ nsresult UpgradeSchemaFrom1_0To2_0(mozIS
   rv = aConnection->SetSchemaVersion(MakeSchemaVersion(2, 0));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   return NS_OK;
 }
 
+nsresult UpgradeSchemaFrom2_0To3_0(mozIStorageConnection* aConnection) {
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aConnection);
+
+  nsresult rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+      "ALTER TABLE data ADD COLUMN utf16Length INTEGER NOT NULL DEFAULT 0;"));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->ExecuteSimpleSQL(
+      NS_LITERAL_CSTRING("UPDATE data "
+                         "SET utf16Length = utf16Length(value) "
+                         "FROM data);"));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->SetSchemaVersion(MakeSchemaVersion(3, 0));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 nsresult SetDefaultPragmas(mozIStorageConnection* aConnection) {
   MOZ_ASSERT(!NS_IsMainThread());
   MOZ_ASSERT(aConnection);
 
   nsresult rv = aConnection->ExecuteSimpleSQL(
       NS_LITERAL_CSTRING("PRAGMA synchronous = FULL;"));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -506,22 +533,24 @@ nsresult CreateStorageConnection(nsIFile
       }
 
       rv = stmt->Execute();
       if (NS_WARN_IF(NS_FAILED(rv))) {
         return rv;
       }
     } else {
       // This logic needs to change next time we change the schema!
-      static_assert(kSQLiteSchemaVersion == int32_t((2 << 4) + 0),
+      static_assert(kSQLiteSchemaVersion == int32_t((3 << 4) + 0),
                     "Upgrade function needed due to schema version increase.");
 
       while (schemaVersion != kSQLiteSchemaVersion) {
         if (schemaVersion == MakeSchemaVersion(1, 0)) {
           rv = UpgradeSchemaFrom1_0To2_0(connection);
+        } else if (schemaVersion == MakeSchemaVersion(2, 0)) {
+          rv = UpgradeSchemaFrom2_0To3_0(connection);
         } else {
           LS_WARNING(
               "Unable to open LocalStorage database, no upgrade path is "
               "available!");
           return NS_ERROR_FAILURE;
         }
 
         if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -3758,29 +3787,36 @@ nsresult WriteOptimizer::PerformWrites(C
 
 nsresult WriteOptimizer::AddItemInfo::Perform(Connection* aConnection,
                                               bool aShadowWrites) {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(aConnection);
 
   Connection::CachedStatement stmt;
   nsresult rv = aConnection->GetCachedStatement(
-      NS_LITERAL_CSTRING("INSERT OR REPLACE INTO data (key, value) "
-                         "VALUES(:key, :value)"),
+      NS_LITERAL_CSTRING(
+          "INSERT OR REPLACE INTO data (key, value, utf16Length) "
+          "VALUES(:key, :value, :utf16Length)"),
       &stmt);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
+  rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("value"), mValue);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("utf16Length"),
+                             mValue.UTF16Length());
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   rv = stmt->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -3815,17 +3851,17 @@ nsresult WriteOptimizer::AddItemInfo::Pe
     return rv;
   }
 
   rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
+  rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("value"), mValue);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   rv = stmt->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -4889,44 +4925,47 @@ void Datastore::SetItem(Database* aDatab
 
   if (oldValue != aValue) {
     bool isNewItem = oldValue.IsVoid();
 
     NotifySnapshots(aDatabase, aKey, oldValue, /* affectsOrder */ isNewItem);
 
     mValues.Put(aKey, aValue);
 
-    int64_t sizeOfItem;
+    int64_t delta;
 
     if (isNewItem) {
       mWriteOptimizer.AddItem(aKey, aValue);
 
       int64_t sizeOfKey = static_cast<int64_t>(aKey.Length());
-      sizeOfItem = sizeOfKey + static_cast<int64_t>(aValue.Length());
-
-      mUpdateBatchUsage += sizeOfItem;
+
+      delta = sizeOfKey + static_cast<int64_t>(aValue.UTF16Length());
+
+      mUpdateBatchUsage += delta;
 
       mSizeOfKeys += sizeOfKey;
-      mSizeOfItems += sizeOfItem;
+      mSizeOfItems += sizeOfKey + static_cast<int64_t>(aValue.Length());
+      ;
     } else {
       mWriteOptimizer.UpdateItem(aKey, aValue);
 
-      sizeOfItem = static_cast<int64_t>(aValue.Length()) -
-                   static_cast<int64_t>(oldValue.Length());
-
-      mUpdateBatchUsage += sizeOfItem;
-
-      mSizeOfItems += sizeOfItem;
+      delta = static_cast<int64_t>(aValue.UTF16Length()) -
+              static_cast<int64_t>(oldValue.UTF16Length());
+
+      mUpdateBatchUsage += delta;
+
+      mSizeOfItems += static_cast<int64_t>(aValue.Length()) -
+                      static_cast<int64_t>(oldValue.Length());
     }
 
     if (IsPersistent()) {
       if (oldValue.IsVoid()) {
-        mConnection->AddItem(aKey, aValue, sizeOfItem);
+        mConnection->AddItem(aKey, aValue, delta);
       } else {
-        mConnection->UpdateItem(aKey, aValue, sizeOfItem);
+        mConnection->UpdateItem(aKey, aValue, delta);
       }
     }
   }
 
   NotifyObservers(aDatabase, aDocumentURI, aKey, aOldValue, aValue);
 }
 
 void Datastore::RemoveItem(Database* aDatabase, const nsString& aDocumentURI,
@@ -4942,60 +4981,61 @@ void Datastore::RemoveItem(Database* aDa
   if (!oldValue.IsVoid()) {
     NotifySnapshots(aDatabase, aKey, oldValue, /* aAffectsOrder */ true);
 
     mValues.Remove(aKey);
 
     mWriteOptimizer.RemoveItem(aKey);
 
     int64_t sizeOfKey = static_cast<int64_t>(aKey.Length());
-    int64_t sizeOfItem = sizeOfKey + static_cast<int64_t>(oldValue.Length());
-
-    mUpdateBatchUsage -= sizeOfItem;
+
+    int64_t delta = -sizeOfKey - static_cast<int64_t>(oldValue.UTF16Length());
+
+    mUpdateBatchUsage += delta;
 
     mSizeOfKeys -= sizeOfKey;
-    mSizeOfItems -= sizeOfItem;
+    mSizeOfItems -= sizeOfKey + static_cast<int64_t>(oldValue.Length());
 
     if (IsPersistent()) {
-      mConnection->RemoveItem(aKey, -sizeOfItem);
+      mConnection->RemoveItem(aKey, delta);
     }
   }
 
   NotifyObservers(aDatabase, aDocumentURI, aKey, aOldValue, VoidLSValue());
 }
 
 void Datastore::Clear(Database* aDatabase, const nsString& aDocumentURI) {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aDatabase);
   MOZ_ASSERT(!mClosed);
   MOZ_ASSERT(mInUpdateBatch);
 
   if (mValues.Count()) {
-    int64_t sizeOfItems = 0;
+    int64_t delta = 0;
     for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
       const nsAString& key = iter.Key();
       const LSValue& value = iter.Data();
 
-      sizeOfItems += (static_cast<int64_t>(key.Length()) +
-                      static_cast<int64_t>(value.Length()));
+      delta += -static_cast<int64_t>(key.Length()) -
+               static_cast<int64_t>(value.UTF16Length());
 
       NotifySnapshots(aDatabase, key, value, /* aAffectsOrder */ true);
     }
 
     mValues.Clear();
 
     mWriteOptimizer.Clear();
 
-    mUpdateBatchUsage -= sizeOfItems;
+    mUpdateBatchUsage += delta;
 
     mSizeOfKeys = 0;
     mSizeOfItems = 0;
 
     if (IsPersistent()) {
-      mConnection->Clear(-sizeOfItems);
+      mConnection->Clear(delta);
     }
   }
 
   NotifyObservers(aDatabase, aDocumentURI, VoidString(), VoidLSValue(),
                   VoidLSValue());
 }
 
 void Datastore::PrivateBrowsingClear() {
@@ -6657,18 +6697,18 @@ nsresult PrepareDatastoreOp::DatabaseWor
       return NS_ERROR_FILE_NO_DEVICE_SPACE;
     }
 
     mozStorageTransaction transaction(
         connection, false, mozIStorageConnection::TRANSACTION_IMMEDIATE);
 
     nsCOMPtr<mozIStorageStatement> stmt;
     rv = connection->CreateStatement(
-        NS_LITERAL_CSTRING("INSERT INTO data (key, value) "
-                           "SELECT key, value "
+        NS_LITERAL_CSTRING("INSERT INTO data (key, value, utf16Length) "
+                           "SELECT key, value, utf16Length(value) "
                            "FROM webappsstore2 "
                            "WHERE originKey = :originKey "
                            "AND originAttributes = :originAttributes;"
 
                            ),
         getter_AddRefs(stmt));
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
@@ -7279,48 +7319,54 @@ nsresult PrepareDatastoreOp::LoadDataOp:
              NestedState::DatabaseWorkLoadData);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
       !MayProceedOnNonOwningThread()) {
     return NS_ERROR_FAILURE;
   }
 
   Connection::CachedStatement stmt;
-  nsresult rv =
-      mConnection->GetCachedStatement(NS_LITERAL_CSTRING("SELECT key, value "
-                                                         "FROM data;"),
-                                      &stmt);
+  nsresult rv = mConnection->GetCachedStatement(
+      NS_LITERAL_CSTRING("SELECT key, value, utf16Length "
+                         "FROM data;"),
+      &stmt);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   bool hasResult;
   while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&hasResult)) && hasResult) {
     nsString key;
     rv = stmt->GetString(0, key);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
-    nsString buffer;
-    rv = stmt->GetString(1, buffer);
+    nsCString buffer;
+    rv = stmt->GetUTF8String(1, buffer);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
-    LSValue value(buffer);
+    int32_t utf16Length;
+    rv = stmt->GetInt32(2, &utf16Length);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    LSValue value(buffer, utf16Length);
 
     mPrepareDatastoreOp->mValues.Put(key, value);
     auto item = mPrepareDatastoreOp->mOrderedItems.AppendElement();
     item->key() = key;
     item->value() = value;
     mPrepareDatastoreOp->mSizeOfKeys += key.Length();
     mPrepareDatastoreOp->mSizeOfItems += key.Length() + value.Length();
 #ifdef DEBUG
-    mPrepareDatastoreOp->mDEBUGUsage += key.Length() + value.Length();
+    mPrepareDatastoreOp->mDEBUGUsage += key.Length() + value.UTF16Length();
 #endif
   }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   return NS_OK;
 }
--- a/dom/localstorage/LSSnapshot.cpp
+++ b/dom/localstorage/LSSnapshot.cpp
@@ -77,17 +77,17 @@ nsresult LSSnapshot::Init(const nsAStrin
     const LSItemInfo& itemInfo = itemInfos[i];
 
     const LSValue& value = itemInfo.value();
 
     if (loadState != LoadState::AllOrderedItems && !value.IsVoid()) {
       mLoadedItems.PutEntry(itemInfo.key());
     }
 
-    mValues.Put(itemInfo.key(), value);
+    mValues.Put(itemInfo.key(), value.AsString());
   }
 
   if (loadState == LoadState::Partial) {
     if (aInitInfo.addKeyToUnknownItems()) {
       mUnknownItems.PutEntry(aKey);
     }
     mInitLength = aInitInfo.totalLength();
     mLength = mInitLength;
@@ -438,32 +438,32 @@ nsresult LSSnapshot::GetItemInternal(con
       } else {
         LSValue value;
         nsTArray<LSItemInfo> itemInfos;
         if (NS_WARN_IF(!mActor->SendLoadValueAndMoreItems(
                 nsString(aKey), &value, &itemInfos))) {
           return NS_ERROR_FAILURE;
         }
 
-        result = value;
+        result = value.AsString();
 
         if (result.IsVoid()) {
           mUnknownItems.PutEntry(aKey);
         } else {
           mLoadedItems.PutEntry(aKey);
           mValues.Put(aKey, result);
 
           // mLoadedItems.Count()==mInitLength is checked below.
         }
 
         for (uint32_t i = 0; i < itemInfos.Length(); i++) {
           const LSItemInfo& itemInfo = itemInfos[i];
 
           mLoadedItems.PutEntry(itemInfo.key());
-          mValues.Put(itemInfo.key(), itemInfo.value());
+          mValues.Put(itemInfo.key(), itemInfo.value().AsString());
         }
 
         if (mLoadedItems.Count() == mInitLength) {
           mLoadedItems.Clear();
           mUnknownItems.Clear();
           mLength = 0;
           mLoadState = LoadState::AllUnorderedItems;
         }
@@ -486,30 +486,30 @@ nsresult LSSnapshot::GetItemInternal(con
         if (result.IsVoid()) {
           LSValue value;
           nsTArray<LSItemInfo> itemInfos;
           if (NS_WARN_IF(!mActor->SendLoadValueAndMoreItems(
                   nsString(aKey), &value, &itemInfos))) {
             return NS_ERROR_FAILURE;
           }
 
-          result = value;
+          result = value.AsString();
 
           MOZ_ASSERT(!result.IsVoid());
 
           mLoadedItems.PutEntry(aKey);
           mValues.Put(aKey, result);
 
           // mLoadedItems.Count()==mInitLength is checked below.
 
           for (uint32_t i = 0; i < itemInfos.Length(); i++) {
             const LSItemInfo& itemInfo = itemInfos[i];
 
             mLoadedItems.PutEntry(itemInfo.key());
-            mValues.Put(itemInfo.key(), itemInfo.value());
+            mValues.Put(itemInfo.key(), itemInfo.value().AsString());
           }
 
           if (mLoadedItems.Count() == mInitLength) {
             mLoadedItems.Clear();
             MOZ_ASSERT(mLength == 0);
             mLoadState = LoadState::AllOrderedItems;
           }
         }
--- a/dom/localstorage/LSValue.cpp
+++ b/dom/localstorage/LSValue.cpp
@@ -5,15 +5,15 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "LSValue.h"
 
 namespace mozilla {
 namespace dom {
 
 const LSValue& VoidLSValue() {
-  static const LSValue sVoidLSValue(VoidString());
+  static const LSValue sVoidLSValue(VoidCString(), 0);
 
   return sVoidLSValue;
 }
 
 }  // namespace dom
 }  // namespace mozilla
--- a/dom/localstorage/LSValue.h
+++ b/dom/localstorage/LSValue.h
@@ -9,45 +9,96 @@
 
 namespace mozilla {
 namespace dom {
 
 /**
  * Represents a LocalStorage value. From content's perspective, values (if
  * present) are always DOMStrings. This is also true from a quota-tracking
  * perspective. However, for memory and disk efficiency it's preferable to store
- * the value in alternate representations. The LSValue type exists to support
- * these alternate representations in future.
+ * the value in alternate utf-8 encoding representations. The LSValue type
+ * exists to support these alternate representations, dynamically re-encoding to
+ * utf-16 while still tracking value size on a utf-16 basis for quota purposes.
  */
 class LSValue final {
   friend struct IPC::ParamTraits<LSValue>;
 
-  nsString mBuffer;
+  nsCString mBuffer;
+  uint32_t mUTF16Length;
 
  public:
-  LSValue() {}
+  LSValue() : mUTF16Length(0) {}
+
+  explicit LSValue(const nsACString& aBuffer, uint32_t aUTF16Length)
+      : mBuffer(aBuffer), mUTF16Length(aUTF16Length) {}
 
-  explicit LSValue(const nsAString& aBuffer) : mBuffer(aBuffer) {}
+  explicit LSValue(const nsAString& aBuffer) : mUTF16Length(aBuffer.Length()) {
+    if (aBuffer.IsVoid()) {
+      mBuffer.SetIsVoid(true);
+    } else {
+      CopyUTF16toUTF8(aBuffer, mBuffer);
+    }
+  }
 
   bool IsVoid() const { return mBuffer.IsVoid(); }
 
   void SetIsVoid(bool aVal) { mBuffer.SetIsVoid(aVal); }
 
+  /**
+   * This represents the "physical" length that the parent process uses for
+   * the size of value/item computation. This can also be used to see how much
+   * memory the value is using at rest or what the cost is for sending the value
+   * over IPC.
+   */
   uint32_t Length() const { return mBuffer.Length(); }
 
+  /*
+   * This represents the "logical" length that content sees and that is also
+   * used for quota management purposes.
+   */
+  uint32_t UTF16Length() const { return mUTF16Length; }
+
   bool Equals(const LSValue& aOther) const {
     return mBuffer == aOther.mBuffer &&
-           mBuffer.IsVoid() == aOther.mBuffer.IsVoid();
+           mBuffer.IsVoid() == aOther.mBuffer.IsVoid() &&
+           mUTF16Length == aOther.mUTF16Length;
   }
 
   bool operator==(const LSValue& aOther) const { return Equals(aOther); }
 
   bool operator!=(const LSValue& aOther) const { return !Equals(aOther); }
 
-  operator const nsString&() const { return mBuffer; }
+  operator const nsCString&() const { return mBuffer; }
+
+  operator Span<const char>() const { return mBuffer; }
+
+  class Converter {
+    nsString mBuffer;
+
+   public:
+    explicit Converter(const LSValue& aValue) {
+      if (aValue.mBuffer.IsVoid()) {
+        mBuffer.SetIsVoid(true);
+      } else {
+        CopyUTF8toUTF16(aValue.mBuffer, mBuffer);
+      }
+    }
+    Converter(Converter&& aOther) : mBuffer(aOther.mBuffer) {}
+    ~Converter() {}
+
+    operator const nsString&() const { return mBuffer; }
+
+   private:
+    Converter() = delete;
+    Converter(const Converter&) = delete;
+    Converter& operator=(const Converter&) = delete;
+    Converter& operator=(const Converter&&) = delete;
+  };
+
+  Converter AsString() const { return Converter(const_cast<LSValue&>(*this)); }
 };
 
 const LSValue& VoidLSValue();
 
 }  // namespace dom
 }  // namespace mozilla
 
 #endif  // mozilla_dom_localstorage_LSValue_h
--- a/dom/localstorage/SerializationHelpers.h
+++ b/dom/localstorage/SerializationHelpers.h
@@ -22,23 +22,26 @@ struct ParamTraits<mozilla::dom::LSSnaps
           mozilla::dom::LSSnapshot::LoadState::EndGuard> {};
 
 template <>
 struct ParamTraits<mozilla::dom::LSValue> {
   typedef mozilla::dom::LSValue paramType;
 
   static void Write(Message* aMsg, const paramType& aParam) {
     WriteParam(aMsg, aParam.mBuffer);
+    WriteParam(aMsg, aParam.mUTF16Length);
   }
 
   static bool Read(const Message* aMsg, PickleIterator* aIter,
                    paramType* aResult) {
-    return ReadParam(aMsg, aIter, &aResult->mBuffer);
+    return ReadParam(aMsg, aIter, &aResult->mBuffer) &&
+           ReadParam(aMsg, aIter, &aResult->mUTF16Length);
   }
 
   static void Log(const paramType& aParam, std::wstring* aLog) {
     LogParam(aParam.mBuffer, aLog);
+    LogParam(aParam.mUTF16Length, aLog);
   }
 };
 
 }  // namespace IPC
 
 #endif  // mozilla_dom_localstorage_SerializationHelpers_h
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -213,17 +213,17 @@ const char kResourceOriginPrefix[] = "re
 #define METADATA_TMP_FILE_NAME ".metadata-tmp"
 #define METADATA_V2_FILE_NAME ".metadata-v2"
 #define METADATA_V2_TMP_FILE_NAME ".metadata-v2-tmp"
 
 #define WEB_APPS_STORE_FILE_NAME "webappsstore.sqlite"
 #define LS_ARCHIVE_FILE_NAME "ls-archive.sqlite"
 #define LS_ARCHIVE_TMP_FILE_NAME "ls-archive-tmp.sqlite"
 
-const uint32_t kLocalStorageArchiveVersion = 1;
+const uint32_t kLocalStorageArchiveVersion = 2;
 
 const char kProfileDoChangeTopic[] = "profile-do-change";
 
 /******************************************************************************
  * SQLite functions
  ******************************************************************************/
 
 int32_t MakeStorageVersion(uint32_t aMajorStorageVersion,
@@ -441,17 +441,16 @@ nsresult LoadLocalStorageArchiveVersion(
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   aVersion = version;
   return NS_OK;
 }
 
-/*
 nsresult SaveLocalStorageArchiveVersion(mozIStorageConnection* aConnection,
                                         uint32_t aVersion) {
   AssertIsOnIOThread();
   MOZ_ASSERT(aConnection);
 
   nsCOMPtr<mozIStorageStatement> stmt;
   nsresult rv = aConnection->CreateStatement(
       NS_LITERAL_CSTRING("UPDATE database SET version = :version;"),
@@ -467,17 +466,16 @@ nsresult SaveLocalStorageArchiveVersion(
 
   rv = stmt->Execute();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   return NS_OK;
 }
-*/
 
 /******************************************************************************
  * Quota manager class declarations
  ******************************************************************************/
 
 }  // namespace
 
 class DirectoryLockImpl final : public DirectoryLock {
@@ -5256,27 +5254,25 @@ nsresult QuotaManager::UpgradeLocalStora
   rv = InitializeLocalStorageArchive(aConnection, 1);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   return NS_OK;
 }
 
-/*
 nsresult QuotaManager::UpgradeLocalStorageArchiveFrom1To2(
     nsCOMPtr<mozIStorageConnection>& aConnection) {
   nsresult rv = SaveLocalStorageArchiveVersion(aConnection, 2);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   return NS_OK;
 }
-*/
 
 #ifdef DEBUG
 
 void QuotaManager::AssertStorageIsInitialized() const {
   AssertIsOnIOThread();
   MOZ_ASSERT(mStorageInitialized);
 }
 
@@ -5490,27 +5486,26 @@ nsresult QuotaManager::EnsureStorageIsIn
         MOZ_ASSERT(version == 0);
 
         rv = InitializeLocalStorageArchive(connection,
                                            kLocalStorageArchiveVersion);
         if (NS_WARN_IF(NS_FAILED(rv))) {
           return rv;
         }
       } else {
-        static_assert(kLocalStorageArchiveVersion == 1,
+        static_assert(kLocalStorageArchiveVersion == 2,
                       "Upgrade function needed due to LocalStorage archive "
                       "version increase.");
 
         while (version != kLocalStorageArchiveVersion) {
           if (version == 0) {
             rv = UpgradeLocalStorageArchiveFrom0To1(connection);
-          } /* else if (version == 1) {
+          } else if (version == 1) {
             rv = UpgradeLocalStorageArchiveFrom1To2(connection);
-          } */
-          else {
+          } else {
             QM_WARNING(
                 "Unable to initialize LocalStorage archive, no upgrade path is "
                 "available!");
             return NS_ERROR_FAILURE;
           }
 
           if (NS_WARN_IF(NS_FAILED(rv))) {
             return rv;
--- a/dom/quota/QuotaManager.h
+++ b/dom/quota/QuotaManager.h
@@ -435,20 +435,18 @@ class QuotaManager final : public Backgr
       nsCOMPtr<mozIStorageConnection>& aConnection);
 
   nsresult DowngradeLocalStorageArchive(
       nsCOMPtr<mozIStorageConnection>& aConnection);
 
   nsresult UpgradeLocalStorageArchiveFrom0To1(
       nsCOMPtr<mozIStorageConnection>& aConnection);
 
-  /*
-    nsresult UpgradeLocalStorageArchiveFrom1To2(
-        nsCOMPtr<mozIStorageConnection>& aConnection);
-  */
+  nsresult UpgradeLocalStorageArchiveFrom1To2(
+      nsCOMPtr<mozIStorageConnection>& aConnection);
 
   nsresult InitializeRepository(PersistenceType aPersistenceType);
 
   nsresult InitializeOrigin(PersistenceType aPersistenceType,
                             const nsACString& aGroup, const nsACString& aOrigin,
                             int64_t aAccessTime, bool aPersisted,
                             nsIFile* aDirectory);
 
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8883ea4630468f1c63076254109d53429752b8ca
GIT binary patch
literal 7792
zc$}qJ2{_bk_a19xm$H7mWM2loFUqb)NSNu%*q0fE8cQ={iEN1?*#@NvNko#e3?&q)
z?4ixBh#0aC5&DMr{lCp5%=Dk@dgi*Gxt_W2bI$Mno#&hbHm0NJ0RR9@0A&lTsl&Br
zG<#`@+fW+7E`Sig&jk+i_C}%5UPu=p>NLU~?P7kM89*1r|ID_CxP1Is05tRj8USD&
zWa3rp$z2eh9Ro&iIaw=qAU%Ii?u|p-d4{*(Z{$IFxiaM#_v^T)QEK8@dBB+7Z#-2|
zfg-h~tpmB4xK$hGM_;wCrUtfL7g<1x^i^xf&pOK;J#3X~8{G#Y1faEyZc5*=Fx$KI
z#W-k4*Un~c^_FmUf#+=FN5XA4?L2$H)}Q4a^Ct;eQi|yX6L$a64osYktoIK5IzHX8
z{AsX|Am88gd~mdc74x{GeDOxv+eryQeA%SeTw*uAEFdmXWLJG9VF*0dQx;WPbe&Z#
z<I3V(jTlNQ<cw$@a%92ub!@t(O&Tn#N%hh)cnJ(~5V}$AJLow%1*|vg7GLShS`Dk*
zLqKyrUxcrqtG|iKrHA0ty`W|7R~NDtFD!PC(QRXWNb{%XNR*32tN|>3xXwRmV%Xj9
zw)~4;xdI0Q?%-#{2~11r5)$R35?dxE+8ho4DjU<@iNNVBR0Jn`SuFbA#`Pe|2FtHU
zrilrjhlhL|F->+iPklQ*_W1H*h1+WIs^{voL%+~=F>D#l9zJ3#7@>EKcMyWZ-tJ8+
z67nl7AzJWPi@b`(VAs6(Eh`ACLj*(}+sqzYg1?<|ZQ`rt0X7YNRi?V983nCiqTY6{
zgB>L5jTD@TsJAtt!CYMw*F}aU3Jf5yL1bVu#h#71vd}@Z_J3LcKM4W?r2s=Zxw-pp
z(Cvd?=(g>t=`KA!z(8+`;p@4cUG+UTm<4XCRa&~JJj<c)F)2@gLOVh|I)Ku=UD^>|
zqr3dQS#~Q(KDXfrcJQ6g2=V=%mYU|bqrg7l0!Mh;ZBwrF!yVGGS-hek$Z>^k>$%2W
zO`wg$QpL$R{}C&XzK2T!CWkHh8@7q|lj_Dh3uA(U)3-jOTW2t){OW_<<?~}o?m0$=
zSVbG7uLmCaB*Z<h)YBO+?HDpa@^yW4cXKU(zBjQ~D~1aA3KW+Y*@u+yx27*FNiAM^
zn!E7eosM?1VqD>D>zT4Yg<@aSIVS1k0Z-I-1@5pq7VM*TV5gs4O|XW%xkvBgHu-2s
zVW9oc;Dmm2_cBr=$>(xst$$^e)jiRQzMf-*W5=wzhTF5IUi>9$65HNCY&|r%5Du&Q
zoY5C+k8Wx4IxJYa*q?syYYENPUOLtZH+gmJR;1k~8qm8?od1lLtl~E3zABo3Y*-Qb
zp)=*5bjC@d5@(kaFdqb3@n3OlqPJfoN&N!J4UP7elaurF^8<Q#1-K&+FrXLGbz6cT
zg663eb(JCR_7Rs7<pe+v7c|TnhKBvi^kJjUog6(1%T{V&FP=Pkl7@9iPQ_qyuQ(*J
z4@J8qCGl8};;&q}f){Q4*(BL)lMrd*PKUVu{0);7eD_ggRVZUQYBql#3dv2FxzrwJ
z`yeJUkyb8fWo8Dc<;=gugOH`%C<Ka%XHg2j3SuViScyxKDh?kM@!VEUqdp=(!b1V<
zKQZ1%muh9Ly)k?MfC^QNYk~eb)%}8ZBB|AQP?zULzxWCli&kWS%;VZ~4JjuHw;F4n
zs{rA+inmO7nWG&KB25HLB%y%`Z^A=vesyIMo-fC!mfYZbTt9Z!-%Ik$B6pu!Q{!s-
zr_+%JFdlI1DKOyR0E<{?IlXiZt3<vYR7y;_J0lCwoP7DtozOdX&sjsD$rVDSLplSk
zJF;y%e2|2Mj(e>ENgaQ&IB4X+ux^6blqh!>u8?Aul4b|h)$#GOYU&rtPYJ6|#Vj#n
zUw7+(ge{?Ty-NF|EQN}Nzm#~)Vmni{pDCca_uog@y5BH!@+`l{*=7F(dCQpPWZt}y
z)Bsz+xTZFtPR7rzrgpj#Ge1(&uA)`n_1@xWLuo*oZJ}9^kGp2?8>!XyF|o*mcMFVh
z6XMDfBc*DmKVFBZ8PQ%C39NsH)?K;86_}|W=rev9sng-_?|KO&cE;eiJ~v?x$WT17
z>?y5G0VXj=2o+rP;IXC<J63Q1%-Iap+T23D>EW5#L(OTe!{3G1kD4a)_}F#C*4%%S
z4auh+UH;C%#T_-228I$I&WTKQ1rg7@8{8sw?cH1vF8(kNZ-fib$;*QZfp=e=gGvy0
zK;n|844jHYBUXJWEG$f;R7|<-4=HoS13J7fcQ;7O$lPy#XRwFiCsSI`1VwS;PGjqL
z?U$xdlwgR~(6D*mhKdjG8M-wlF)~8?@F2g01Os$w>E+EWl8Yo|#{70-k)s&@cautz
zO#UeLpQNHpF&m``CB?+JR+lr<6%!1~9FTZ@N$JK7c$>AE>^Vr*#p(RRum1ZUm(PgF
zQ5Aepc`;y0ExP*P)IggF+isq4HOK<&gx{b1DQ9Y162pTUKe3vf!<iju(PLx^a{&P!
z(7&wFIKmw+ZhGEnUx+?qP9qD0IgNQ<sCw@4qzg%idc}bgN9%cg(t1o!ThCgCBLxCa
zl%5p8Md2R6Q~3-er>^_FQjJ6O<a>GwzH#c=7ia8mRC%$o&d)bO#2U(s59>c!tmMqr
z)AOL2t+*>B>eTR~({7!Xz)XWfp0Dv@ZN5P%iRuUL4M$&6rzz`nwD=GqW%}^YUg5d)
zv~q1%RpCL*T>?5n+(C$6)NeM(PDNnem4K<*uO6n?t>uFi`2-TxGLf4!i9#9SAwwFP
zcE*WEv9IyyPPJl8!J#tjQfff+H*Ml89|F?b4-wJ33KCn?h)L{`uXk_YA1R1P<@eQJ
zNqLb|myyEQ`3;{vGV{v2{kz#8N5vABeRhiH<FEO0(<cjfSTKry`OeEJnP9lee8ybl
z;$&#Xy+gz^LWP@aT6A+2AiSJl2sba(_C?VTflWBrH0@o#hqt_oe?U`_M3%i7>FDTO
zO@one)5F21ndA4yLyjGD%r1=Mr2EMXB<*aa7;&dYTnbc~K~Vx(vqWH^sou=Ass5!R
zkOoP=@5qszcjpLP@Ga7ZI3<SscJh#+5dC*khYI8Bm&|`s#}-+cjEnk8Btc=0Z%p6+
z-2Lb`P_5A-M|2jClU)VZ98b<@?$eUC77$eNQovMPXSEy0Hy!D~&Mb`TaZBZKle@DY
zDpQ9(8T<DuDfZ7_*HzMtUb2(*=11x>-4&&MX&kDlnVGF(hzCjLd@~+$%&fA0tMaN;
z;+1h%;=J6U0)IMNvr~1^?upm~dflRUupXw#qXf}d+E)7ZKZ2MB*7PD{^aLGmHBLYS
zisvkwsFk&UrF5b;0A(>;HF&%&?vZQI^8<LttkEtT-&1p()$g%ys|Mkf(W0MfW`bnU
z;lRfDmf+Vc$!3yaH{QYe7tctWJv4CQV*65gOUFyu3WGtgo;`i+?ordk?&O3*)lXS=
zA;XG!Esn1_V#E8Yec`5Y#a)8x)4*Jj7j|aOfkt&D2DOcH^BH4vZ#2srmXC&i2(sqc
zS*LMR9`m_cFY;g}pHz+^ES-mS0(7zXjkQ9|S%GwrEnj@~x9P>!mDVWqs)+8|DZd7B
zuWON~dGraY6Yq@Uo4z~5XM&~snDu*?<Cp2|D>xz!P6uexk}IK_t8Q6&P0++fjs1rb
zD3}Vdu+P=Y8<`t6DmIwNLzGT9!X2HRy}@8d;^#cPCzP3Kn(i>87M+~^#V&{v6Ag(3
z87#4;RcRh9kwmOT+BL}w0i2L7Ftm$<GaPslwN5KQd|?DOzBC>X6SG<+*2)D7OD!Lf
zBOe|;rD0|k4Le^lSaK*OzyoyccvOXQ&WhjlSG%v4mqcf*^v$9UGy{akxsvd)_V3uj
z*uk*uWbwP9Lg4VJw_cZ_Q=hH#3I-A)b8V8cXQYiLArEmrs!SvJBI8Q)lL|U*6*JJD
zq|TD2ef+V|r}xUDONw>*-1Of^z!TH19#>}28!0t^gc<JqVjCK0`6qomN8)oCcL84M
zjPH3O>^s^XnR&#-sNMrF$9uSf@3);tq;yJK8YRl3j?hi|+v--KJpXua+_BdJ*FIvq
z|MtVv;R2%)OI8W9MbF`L+Wen2k4-AveKlEH^T+;_%ms7n)qn)CXDne8ms7uklrCsy
z$Y*&by%91@&=F0*shpoo;1uNh%g{+C5*Ov9F_L-l;&6L&Vu+u`;?8hfPGn~1IDCnB
zcbwhi%gHhp{pl-+0yT^??_s%{_(T6?EySA(-;wp?=1@XRXM9K)6Upa|N!Hg5kVITE
zl=>R0M5kpoBxa{vwPVM~I1AXAhE|wvvk4>LziIMT*xL6m=Zy@1D{DheTm#;0l=OeV
zoPY^EZzA1~!4kvYgQZ`hZ8laSHM58W?H5C)?Ty_yk03SnJw4zT(+TQPrN~gl7&fAA
zI;q%-Y_mB9X@tTlCnKo?C}SlbMUb-YVWggQ{o1EKwz6cg&8M7r{hFuDQS#9PsiPb_
zsNwtP;-`$1eEL928b%c<Sp`6SeD{*Y7sI?MHF?uZnUCZxFR71Hlo3&_S@O=7lreB8
zb&S8MS4pyXGO<uZ`kPvnBXiJ=VlJw5O5V_t627IF@c*e%QL>o!ux(09+D}pD9eFoJ
z>YW-p1vFIale}#r<*TBK?>E&cP8LZ64hkrLQ=2km&Sg``M3pkh`z2C3E>6nmDAwg>
zr8z07AU9w`p}sxl#;W^o%r(D4!;*-Y<TW_M-!=FKL<UB{dL;M&OT<Uuxb`F9f6zVp
ARsaA1
new file mode 100644
--- /dev/null
+++ b/dom/quota/test/unit/test_localStorageArchive2upgrade.js
@@ -0,0 +1,65 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+/**
+ * This test is mainly to verify that local storage directories are not removed
+ * during local storage archive upgrade from version 1 to version 2.
+ * See bug 1546310.
+ */
+
+async function testSteps() {
+  const lsDirs = [
+    "storage/default/http+++example.com/ls",
+    "storage/default/http+++localhost/ls",
+    "storage/default/http+++www.mozilla.org/ls",
+  ];
+
+  info("Clearing");
+
+  let request = clear();
+  await requestFinished(request);
+
+  info("Installing package");
+
+  // The profile contains three initialized origin directories with local
+  // storage data, local storage archive, a script for origin initialization,
+  // the storage database and the web apps store database:
+  // - storage/default/https+++example.com
+  // - storage/default/https+++localhost
+  // - storage/default/https+++www.mozilla.org
+  // - storage/ls-archive.sqlite
+  // - create_db.js
+  // - storage.sqlite
+  // - webappsstore.sqlite
+  // The file create_db.js in the package was run locally (with a build with
+  // local storage archive version 1), specifically it was temporarily added to
+  // xpcshell.ini and then executed:
+  //   mach xpcshell-test --interactive dom/localstorage/test/unit/create_db.js
+  // Note: to make it become the profile in the test, additional manual steps
+  // are needed.
+  // 1. Remove the folder "storage/temporary".
+  installPackage("localStorageArchive2upgrade_profile");
+
+  info("Checking ls dirs");
+
+  for (let lsDir of lsDirs) {
+    let dir = getRelativeFile(lsDir);
+
+    exists = dir.exists();
+    ok(exists, "ls directory does exist");
+  }
+
+  request = init();
+  request = await requestFinished(request);
+
+  info("Checking ls dirs");
+
+  for (let lsDir of lsDirs) {
+    let dir = getRelativeFile(lsDir);
+
+    exists = dir.exists();
+    ok(exists, "ls directory does exist");
+  }
+}
--- a/dom/quota/test/unit/xpcshell.ini
+++ b/dom/quota/test/unit/xpcshell.ini
@@ -9,16 +9,17 @@ support-files =
   clearStorageForPrincipal_profile.zip
   createLocalStorage_profile.zip
   defaultStorageUpgrade_profile.zip
   getUsage_profile.zip
   groupMismatch_profile.zip
   idbSubdirUpgrade1_profile.zip
   idbSubdirUpgrade2_profile.zip
   localStorageArchive1upgrade_profile.zip
+  localStorageArchive2upgrade_profile.zip
   localStorageArchiveDowngrade_profile.zip
   morgueCleanup_profile.zip
   obsoleteOriginAttributes_profile.zip
   originAttributesUpgrade_profile.zip
   removeLocalStorage1_profile.zip
   removeLocalStorage2_profile.zip
   storagePersistentUpgrade_profile.zip
   tempMetadataCleanup_profile.zip
@@ -31,16 +32,17 @@ support-files =
 [test_clearStorageForPrincipal.js]
 [test_defaultStorageUpgrade.js]
 [test_getUsage.js]
 [test_groupMismatch.js]
 [test_idbSubdirUpgrade.js]
 [test_initTemporaryStorage.js]
 [test_listInitializedOrigins.js]
 [test_localStorageArchive1upgrade.js]
+[test_localStorageArchive2upgrade.js]
 [test_localStorageArchiveDowngrade.js]
 [test_morgueCleanup.js]
 [test_obsoleteOriginAttributesUpgrade.js]
 [test_obsoleteOrigins.js]
 [test_originAttributesUpgrade.js]
 [test_persist.js]
 [test_persist_eviction.js]
 [test_persist_globalLimit.js]