Bug 964561 - Part 6: Core changes for storing structured clone data outside of the database; r=asuth
authorJan Varga <jan.varga@gmail.com>
Tue, 25 Oct 2016 21:18:22 +0200
changeset 362298 34a3a5034620002d2dd74bbf82f75e96d2f51ef4
parent 362297 94f3aca1fe730e847fd17d45624999e626088768
child 362299 34b5720fa068f1ce7585d5de803dadd389849f25
push id6795
push userjlund@mozilla.com
push dateMon, 23 Jan 2017 14:19:46 +0000
treeherdermozilla-beta@76101b503191 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersasuth
bugs964561
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 964561 - Part 6: Core changes for storing structured clone data outside of the database; r=asuth
dom/indexedDB/ActorsChild.cpp
dom/indexedDB/ActorsParent.cpp
dom/indexedDB/IndexedDatabase.h
dom/indexedDB/PBackgroundIDBSharedTypes.ipdlh
--- a/dom/indexedDB/ActorsChild.cpp
+++ b/dom/indexedDB/ActorsChild.cpp
@@ -664,16 +664,25 @@ ConvertActorsToBlobs(IDBDatabase* aDatab
 
             default:
               MOZ_CRASH("Should never get here!");
           }
 
           break;
         }
 
+        case BlobOrMutableFile::Tnull_t: {
+          StructuredCloneFile* file = aFiles.AppendElement();
+          MOZ_ASSERT(file);
+
+          file->mType = StructuredCloneFile::eStructuredClone;
+
+          break;
+        }
+
         default:
           MOZ_CRASH("Should never get here!");
       }
     }
   }
 }
 
 void
--- a/dom/indexedDB/ActorsParent.cpp
+++ b/dom/indexedDB/ActorsParent.cpp
@@ -4114,18 +4114,19 @@ UpgradeSchemaFrom23_0To24_0(mozIStorageC
   }
 
   return NS_OK;
 }
 
 nsresult
 UpgradeSchemaFrom24_0To25_0(mozIStorageConnection* aConnection)
 {
-  // The only change between 24 and 25 was a different structured clone format,
-  // but it's backwards-compatible.
+  // The changes between 24 and 25 were an upgraded snappy library, a different
+  // structured clone format and a different file_ds format. But everything is
+  // backwards-compatible.
   nsresult rv = aConnection->SetSchemaVersion(MakeSchemaVersion(25, 0));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   return NS_OK;
 }
 
@@ -6001,20 +6002,26 @@ private:
                                        uint32_t aDataIndex,
                                        uint32_t aFileIdsIndex,
                                        FileManager* aFileManager,
                                        StructuredCloneReadInfo* aInfo);
 
   static nsresult
   GetStructuredCloneReadInfoFromBlob(const uint8_t* aBlobData,
                                      uint32_t aBlobDataLength,
+                                     FileManager* aFileManager,
                                      const nsAString& aFileIds,
-                                     FileManager* aFileManager,
                                      StructuredCloneReadInfo* aInfo);
 
+  static nsresult
+  GetStructuredCloneReadInfoFromExternalBlob(uint64_t aIntData,
+                                             FileManager* aFileManager,
+                                             const nsAString& aFileIds,
+                                             StructuredCloneReadInfo* aInfo);
+
   // Not to be overridden by subclasses.
   NS_DECL_MOZISTORAGEPROGRESSHANDLER
 };
 
 class MOZ_STACK_CLASS DatabaseOperationBase::AutoSetProgressHandler final
 {
   mozIStorageConnection* mConnection;
 #ifdef DEBUG
@@ -8146,16 +8153,17 @@ class ObjectStoreAddOrPutRequestOp final
   RefPtr<FileManager> mFileManager;
 
   Key mResponse;
   const nsCString mGroup;
   const nsCString mOrigin;
   const PersistenceType mPersistenceType;
   const bool mOverwrite;
   bool mObjectStoreMayHaveIndexes;
+  bool mDataOverThreshold;
 
 private:
   // Only created by TransactionBase.
   ObjectStoreAddOrPutRequestOp(TransactionBase* aTransaction,
                                const RequestParams& aParams);
 
   ~ObjectStoreAddOrPutRequestOp()
   { }
@@ -8180,16 +8188,17 @@ private:
 };
 
 struct ObjectStoreAddOrPutRequestOp::StoredFileInfo final
 {
   enum Type
   {
     eBlob,
     eMutableFile,
+    eStructuredClone
   };
 
   RefPtr<DatabaseFile> mFileActor;
   RefPtr<FileInfo> mFileInfo;
   nsCOMPtr<nsIInputStream> mInputStream;
   Type mType;
   bool mCopiedSuccessfully;
 
@@ -8220,16 +8229,21 @@ struct ObjectStoreAddOrPutRequestOp::Sto
       case eBlob:
         aText.AppendInt(id);
         break;
 
       case eMutableFile:
         aText.AppendInt(-id);
         break;
 
+      case eStructuredClone:
+        aText.Append('.');
+        aText.AppendInt(id);
+        break;
+
       default:
         MOZ_CRASH("Should never get here!");
     }
   }
 };
 
 class ObjectStoreAddOrPutRequestOp::SCInputStream final
   : public nsIInputStream
@@ -9569,25 +9583,31 @@ DeserializeStructuredCloneFile(FileManag
 {
   MOZ_ASSERT(!aText.IsEmpty());
   MOZ_ASSERT(aFile);
 
   nsresult rv;
   int32_t id;
   StructuredCloneFile::Type type;
 
-  if (aText.First() == '-') {
+  bool isDot = false;
+  if ((isDot = aText.First() == '.') ||
+      aText.First() == '-') {
     nsString text(Substring(aText, 1));
 
     id = text.ToInteger(&rv);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
-    type = StructuredCloneFile::eMutableFile;
+    if (isDot) {
+      type = StructuredCloneFile::eStructuredClone;
+    } else {
+      type = StructuredCloneFile::eMutableFile;
+    }
   } else {
     id = aText.ToInteger(&rv);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
     type = StructuredCloneFile::eBlob;
   }
@@ -9749,16 +9769,21 @@ ConvertBlobsToActors(PBackgroundParent* 
 
           MOZ_ALWAYS_TRUE(aActors.AppendElement(NullableMutableFile(actor),
                                                 fallible));
         }
 
         break;
       }
 
+      case StructuredCloneFile::eStructuredClone:
+        MOZ_ALWAYS_TRUE(aActors.AppendElement(null_t(), fallible));
+
+        break;
+
       default:
         MOZ_CRASH("Should never get here!");
     }
   }
 
   return NS_OK;
 }
 
@@ -19087,30 +19112,24 @@ DatabaseOperationBase::GetStructuredClon
                                                  FileManager* aFileManager,
                                                  StructuredCloneReadInfo* aInfo)
 {
   MOZ_ASSERT(!IsOnBackgroundThread());
   MOZ_ASSERT(aSource);
   MOZ_ASSERT(aFileManager);
   MOZ_ASSERT(aInfo);
 
-#ifdef DEBUG
-  {
-    int32_t columnType;
-    MOZ_ASSERT(NS_SUCCEEDED(aSource->GetTypeOfIndex(aDataIndex, &columnType)));
-    MOZ_ASSERT(columnType == mozIStorageStatement::VALUE_TYPE_BLOB);
-  }
-#endif
-
-  const uint8_t* blobData;
-  uint32_t blobDataLength;
-  nsresult rv = aSource->GetSharedBlob(aDataIndex, &blobDataLength, &blobData);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
+  int32_t columnType;
+  nsresult rv = aSource->GetTypeOfIndex(aDataIndex, &columnType);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(columnType == mozIStorageStatement::VALUE_TYPE_BLOB ||
+             columnType == mozIStorageStatement::VALUE_TYPE_INTEGER);
 
   bool isNull;
   rv = aSource->GetIsNull(aFileIdsIndex, &isNull);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   nsString fileIds;
@@ -19119,35 +19138,56 @@ DatabaseOperationBase::GetStructuredClon
     fileIds.SetIsVoid(true);
   } else {
     rv = aSource->GetString(aFileIdsIndex, fileIds);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
   }
 
-  rv = GetStructuredCloneReadInfoFromBlob(blobData,
-                                          blobDataLength,
-                                          fileIds,
-                                          aFileManager,
-                                          aInfo);
+  if (columnType == mozIStorageStatement::VALUE_TYPE_INTEGER) {
+    uint64_t intData;
+    rv = aSource->GetInt64(aDataIndex, reinterpret_cast<int64_t*>(&intData));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = GetStructuredCloneReadInfoFromExternalBlob(intData,
+                                                    aFileManager,
+                                                    fileIds,
+                                                    aInfo);
+  } else {
+    const uint8_t* blobData;
+    uint32_t blobDataLength;
+    nsresult rv =
+      aSource->GetSharedBlob(aDataIndex, &blobDataLength, &blobData);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = GetStructuredCloneReadInfoFromBlob(blobData,
+                                            blobDataLength,
+                                            aFileManager,
+                                            fileIds,
+                                            aInfo);
+  }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   return NS_OK;
 }
 
 // static
 nsresult
 DatabaseOperationBase::GetStructuredCloneReadInfoFromBlob(
                                                  const uint8_t* aBlobData,
                                                  uint32_t aBlobDataLength,
+                                                 FileManager* aFileManager,
                                                  const nsAString& aFileIds,
-                                                 FileManager* aFileManager,
                                                  StructuredCloneReadInfo* aInfo)
 {
   MOZ_ASSERT(!IsOnBackgroundThread());
   MOZ_ASSERT(aFileManager);
   MOZ_ASSERT(aInfo);
 
   PROFILER_LABEL("IndexedDB",
                  "DatabaseOperationBase::GetStructuredCloneReadInfoFromBlob",
@@ -19186,16 +19226,88 @@ DatabaseOperationBase::GetStructuredClon
     }
   }
 
   return NS_OK;
 }
 
 // static
 nsresult
+DatabaseOperationBase::GetStructuredCloneReadInfoFromExternalBlob(
+                                                 uint64_t aIntData,
+                                                 FileManager* aFileManager,
+                                                 const nsAString& aFileIds,
+                                                 StructuredCloneReadInfo* aInfo)
+{
+  MOZ_ASSERT(!IsOnBackgroundThread());
+  MOZ_ASSERT(aFileManager);
+  MOZ_ASSERT(aInfo);
+
+  PROFILER_LABEL(
+            "IndexedDB",
+            "DatabaseOperationBase::GetStructuredCloneReadInfoFromExternalBlob",
+            js::ProfileEntry::Category::STORAGE);
+
+  nsresult rv;
+
+  if (!aFileIds.IsVoid()) {
+    rv = DeserializeStructuredCloneFiles(aFileManager, aFileIds, aInfo->mFiles);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  // Higher and lower 32 bits described
+  // in ObjectStoreAddOrPutRequestOp::DoDatabaseWork.
+  uint32_t index = uint32_t(aIntData);
+
+  if (index >= aInfo->mFiles.Length()) {
+    MOZ_ASSERT(false, "Bad index value!");
+    return NS_ERROR_UNEXPECTED;
+  }
+
+  StructuredCloneFile& file = aInfo->mFiles[index];
+  MOZ_ASSERT(file.mFileInfo);
+  MOZ_ASSERT(file.mType == StructuredCloneFile::eStructuredClone);
+
+  nsCOMPtr<nsIFile> nativeFile = GetFileForFileInfo(file.mFileInfo);
+  if (NS_WARN_IF(!nativeFile)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  nsCOMPtr<nsIInputStream> stream;
+  rv = NS_NewLocalFileInputStream(getter_AddRefs(stream), nativeFile);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  do {
+    char buffer[kFileCopyBufferSize];
+
+    uint32_t numRead;
+    rv = stream->Read(buffer, sizeof(buffer), &numRead);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      break;
+    }
+
+    if (!numRead) {
+      break;
+    }
+
+    if (NS_WARN_IF(!aInfo->mData.WriteBytes(buffer, numRead))) {
+      rv = NS_ERROR_OUT_OF_MEMORY;
+      break;
+    }
+  } while (true);
+
+  return rv;
+}
+
+// static
+nsresult
 DatabaseOperationBase::BindKeyRangeToStatement(
                                             const SerializedKeyRange& aKeyRange,
                                             mozIStorageStatement* aStatement)
 {
   MOZ_ASSERT(!IsOnBackgroundThread());
   MOZ_ASSERT(aStatement);
 
   nsresult rv = NS_OK;
@@ -24377,17 +24489,18 @@ UpdateIndexDataValuesFunction::OnFunctio
     MOZ_ASSERT(valueType == mozIStorageValueArray::VALUE_TYPE_NULL ||
                valueType == mozIStorageValueArray::VALUE_TYPE_BLOB);
 
     MOZ_ALWAYS_SUCCEEDS(aValues->GetTypeOfIndex(2, &valueType));
     MOZ_ASSERT(valueType == mozIStorageValueArray::VALUE_TYPE_NULL ||
                valueType == mozIStorageValueArray::VALUE_TYPE_TEXT);
 
     MOZ_ALWAYS_SUCCEEDS(aValues->GetTypeOfIndex(3, &valueType));
-    MOZ_ASSERT(valueType == mozIStorageValueArray::VALUE_TYPE_BLOB);
+    MOZ_ASSERT(valueType == mozIStorageValueArray::VALUE_TYPE_BLOB ||
+               valueType == mozIStorageValueArray::VALUE_TYPE_INTEGER);
   }
 #endif
 
   StructuredCloneReadInfo cloneInfo;
   nsresult rv =
     GetStructuredCloneReadInfoFromValueArray(aValues,
                                              /* aDataIndex */ 3,
                                              /* aFileIdsIndex */ 2,
@@ -25211,16 +25324,20 @@ ObjectStoreAddOrPutRequestOp::ObjectStor
   MOZ_ASSERT(aParams.type() == RequestParams::TObjectStoreAddParams ||
              aParams.type() == RequestParams::TObjectStorePutParams);
 
   mMetadata =
     aTransaction->GetMetadataForObjectStoreId(mParams.objectStoreId());
   MOZ_ASSERT(mMetadata);
 
   mObjectStoreMayHaveIndexes = mMetadata->HasLiveIndexes();
+
+  mDataOverThreshold =
+    snappy::MaxCompressedLength(mParams.cloneInfo().data().data.Size()) >
+      IndexedDatabaseManager::DataThreshold();
 }
 
 nsresult
 ObjectStoreAddOrPutRequestOp::RemoveOldIndexDataValues(
                                                 DatabaseConnection* aConnection)
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(aConnection);
@@ -25443,16 +25560,33 @@ ObjectStoreAddOrPutRequestOp::Init(Trans
 
         case DatabaseOrMutableFile::TPBackgroundIDBDatabaseFileChild:
         default:
           MOZ_CRASH("Should never get here!");
       }
     }
   }
 
+  if (mDataOverThreshold) {
+    StoredFileInfo* storedFileInfo = mStoredFileInfos.AppendElement(fallible);
+    MOZ_ASSERT(storedFileInfo);
+
+    if (!mFileManager) {
+      mFileManager = aTransaction->GetDatabase()->GetFileManager();
+      MOZ_ASSERT(mFileManager);
+    }
+
+    storedFileInfo->mFileInfo = mFileManager->GetNewFileInfo();
+
+    storedFileInfo->mInputStream =
+      new SCInputStream(mParams.cloneInfo().data().data);
+
+    storedFileInfo->mType = StoredFileInfo::eStructuredClone;
+  }
+
   return true;
 }
 
 nsresult
 ObjectStoreAddOrPutRequestOp::DoDatabaseWork(DatabaseConnection* aConnection)
 {
   MOZ_ASSERT(aConnection);
   aConnection->AssertIsOnConnectionThread();
@@ -25581,39 +25715,50 @@ ObjectStoreAddOrPutRequestOp::DoDatabase
         result = iter.AdvanceAcrossSegments(cloneData, 1);
         MOZ_ASSERT(result);
       }
     }
   }
 
   key.BindToStatement(stmt, NS_LITERAL_CSTRING("key"));
 
-  nsCString flatCloneData;
-  flatCloneData.SetLength(cloneDataSize);
-  auto iter = cloneData.Iter();
-  cloneData.ReadBytes(iter, flatCloneData.BeginWriting(), cloneDataSize);
-
-  // Compress the bytes before adding into the database.
-  const char* uncompressed = flatCloneData.BeginReading();
-  size_t uncompressedLength = cloneDataSize;
-
-  // We don't have a smart pointer class that calls free, so we need to
-  // manage | compressed | manually.
-  {
+  if (mDataOverThreshold) {
+    // Higher 32 bits reserved for flags like compressed/uncompressed. For now,
+    // we don't compress externally stored structured clone data since snappy
+    // doesn't support streaming yet and we don't want to allocate another
+    // large memory buffer for that.
+    // Lower 32 bits used for storing file_ids index.
+    uint64_t data = uint64_t(mStoredFileInfos.Length() - 1);
+
+    rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("data"), data);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  } else {
+    nsCString flatCloneData;
+    flatCloneData.SetLength(cloneDataSize);
+    auto iter = cloneData.Iter();
+    cloneData.ReadBytes(iter, flatCloneData.BeginWriting(), cloneDataSize);
+
+    // Compress the bytes before adding into the database.
+    const char* uncompressed = flatCloneData.BeginReading();
+    size_t uncompressedLength = cloneDataSize;
+
     size_t compressedLength = snappy::MaxCompressedLength(uncompressedLength);
 
-    char* compressed = static_cast<char*>(malloc(compressedLength));
+    UniqueFreePtr<char> compressed(
+      static_cast<char*>(malloc(compressedLength)));
     if (NS_WARN_IF(!compressed)) {
       return NS_ERROR_OUT_OF_MEMORY;
     }
 
-    snappy::RawCompress(uncompressed, uncompressedLength, compressed,
+    snappy::RawCompress(uncompressed, uncompressedLength, compressed.get(),
                         &compressedLength);
 
-    uint8_t* dataBuffer = reinterpret_cast<uint8_t*>(compressed);
+    uint8_t* dataBuffer = reinterpret_cast<uint8_t*>(compressed.release());
     size_t dataBufferLength = compressedLength;
 
     rv = stmt->BindAdoptedBlobByName(NS_LITERAL_CSTRING("data"), dataBuffer,
                                      dataBufferLength);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
   }
@@ -25869,20 +26014,21 @@ ObjectStoreAddOrPutRequestOp::Cleanup()
 {
   AssertIsOnOwningThread();
 
   if (!mStoredFileInfos.IsEmpty()) {
     for (uint32_t count = mStoredFileInfos.Length(), index = 0;
          index < count;
          index++) {
       StoredFileInfo& storedFileInfo = mStoredFileInfos[index];
+
+      MOZ_ASSERT_IF(storedFileInfo.mType == StoredFileInfo::eMutableFile,
+                    !storedFileInfo.mCopiedSuccessfully);
+
       RefPtr<DatabaseFile>& fileActor = storedFileInfo.mFileActor;
-
-      MOZ_ASSERT_IF(!fileActor, !storedFileInfo.mCopiedSuccessfully);
-
       if (fileActor && storedFileInfo.mCopiedSuccessfully) {
         fileActor->ClearInputStream();
       }
     }
 
     mStoredFileInfos.Clear();
   }
 
--- a/dom/indexedDB/IndexedDatabase.h
+++ b/dom/indexedDB/IndexedDatabase.h
@@ -23,16 +23,17 @@ namespace indexedDB {
 class FileInfo;
 class SerializedStructuredCloneReadInfo;
 
 struct StructuredCloneFile
 {
   enum Type {
     eBlob,
     eMutableFile,
+    eStructuredClone,
   };
 
   RefPtr<Blob> mBlob;
   RefPtr<IDBMutableFile> mMutableFile;
   RefPtr<FileInfo> mFileInfo;
   Type mType;
 
   // In IndexedDatabaseInlines.h
--- a/dom/indexedDB/PBackgroundIDBSharedTypes.ipdlh
+++ b/dom/indexedDB/PBackgroundIDBSharedTypes.ipdlh
@@ -49,16 +49,17 @@ struct SerializedKeyRange
 union NullableMutableFile
 {
   null_t;
   PBackgroundMutableFile;
 };
 
 union BlobOrMutableFile
 {
+  null_t;
   PBlob;
   NullableMutableFile;
 };
 
 struct SerializedStructuredCloneReadInfo
 {
   SerializedStructuredCloneBuffer data;
   BlobOrMutableFile[] blobs;