author | Jan Varga <jan.varga@gmail.com> |
Tue, 25 Oct 2016 21:18:22 +0200 | |
changeset 362298 | 34a3a5034620002d2dd74bbf82f75e96d2f51ef4 |
parent 362297 | 94f3aca1fe730e847fd17d45624999e626088768 |
child 362299 | 34b5720fa068f1ce7585d5de803dadd389849f25 |
push id | 6795 |
push user | jlund@mozilla.com |
push date | Mon, 23 Jan 2017 14:19:46 +0000 |
treeherder | mozilla-beta@76101b503191 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | asuth |
bugs | 964561 |
milestone | 52.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/dom/indexedDB/ActorsChild.cpp +++ b/dom/indexedDB/ActorsChild.cpp @@ -664,16 +664,25 @@ ConvertActorsToBlobs(IDBDatabase* aDatab default: MOZ_CRASH("Should never get here!"); } break; } + case BlobOrMutableFile::Tnull_t: { + StructuredCloneFile* file = aFiles.AppendElement(); + MOZ_ASSERT(file); + + file->mType = StructuredCloneFile::eStructuredClone; + + break; + } + default: MOZ_CRASH("Should never get here!"); } } } } void
--- a/dom/indexedDB/ActorsParent.cpp +++ b/dom/indexedDB/ActorsParent.cpp @@ -4114,18 +4114,19 @@ UpgradeSchemaFrom23_0To24_0(mozIStorageC } return NS_OK; } nsresult UpgradeSchemaFrom24_0To25_0(mozIStorageConnection* aConnection) { - // The only change between 24 and 25 was a different structured clone format, - // but it's backwards-compatible. + // The changes between 24 and 25 were an upgraded snappy library, a different + // structured clone format and a different file_ds format. But everything is + // backwards-compatible. nsresult rv = aConnection->SetSchemaVersion(MakeSchemaVersion(25, 0)); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } return NS_OK; } @@ -6001,20 +6002,26 @@ private: uint32_t aDataIndex, uint32_t aFileIdsIndex, FileManager* aFileManager, StructuredCloneReadInfo* aInfo); static nsresult GetStructuredCloneReadInfoFromBlob(const uint8_t* aBlobData, uint32_t aBlobDataLength, + FileManager* aFileManager, const nsAString& aFileIds, - FileManager* aFileManager, StructuredCloneReadInfo* aInfo); + static nsresult + GetStructuredCloneReadInfoFromExternalBlob(uint64_t aIntData, + FileManager* aFileManager, + const nsAString& aFileIds, + StructuredCloneReadInfo* aInfo); + // Not to be overridden by subclasses. NS_DECL_MOZISTORAGEPROGRESSHANDLER }; class MOZ_STACK_CLASS DatabaseOperationBase::AutoSetProgressHandler final { mozIStorageConnection* mConnection; #ifdef DEBUG @@ -8146,16 +8153,17 @@ class ObjectStoreAddOrPutRequestOp final RefPtr<FileManager> mFileManager; Key mResponse; const nsCString mGroup; const nsCString mOrigin; const PersistenceType mPersistenceType; const bool mOverwrite; bool mObjectStoreMayHaveIndexes; + bool mDataOverThreshold; private: // Only created by TransactionBase. ObjectStoreAddOrPutRequestOp(TransactionBase* aTransaction, const RequestParams& aParams); ~ObjectStoreAddOrPutRequestOp() { } @@ -8180,16 +8188,17 @@ private: }; struct ObjectStoreAddOrPutRequestOp::StoredFileInfo final { enum Type { eBlob, eMutableFile, + eStructuredClone }; RefPtr<DatabaseFile> mFileActor; RefPtr<FileInfo> mFileInfo; nsCOMPtr<nsIInputStream> mInputStream; Type mType; bool mCopiedSuccessfully; @@ -8220,16 +8229,21 @@ struct ObjectStoreAddOrPutRequestOp::Sto case eBlob: aText.AppendInt(id); break; case eMutableFile: aText.AppendInt(-id); break; + case eStructuredClone: + aText.Append('.'); + aText.AppendInt(id); + break; + default: MOZ_CRASH("Should never get here!"); } } }; class ObjectStoreAddOrPutRequestOp::SCInputStream final : public nsIInputStream @@ -9569,25 +9583,31 @@ DeserializeStructuredCloneFile(FileManag { MOZ_ASSERT(!aText.IsEmpty()); MOZ_ASSERT(aFile); nsresult rv; int32_t id; StructuredCloneFile::Type type; - if (aText.First() == '-') { + bool isDot = false; + if ((isDot = aText.First() == '.') || + aText.First() == '-') { nsString text(Substring(aText, 1)); id = text.ToInteger(&rv); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } - type = StructuredCloneFile::eMutableFile; + if (isDot) { + type = StructuredCloneFile::eStructuredClone; + } else { + type = StructuredCloneFile::eMutableFile; + } } else { id = aText.ToInteger(&rv); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } type = StructuredCloneFile::eBlob; } @@ -9749,16 +9769,21 @@ ConvertBlobsToActors(PBackgroundParent* MOZ_ALWAYS_TRUE(aActors.AppendElement(NullableMutableFile(actor), fallible)); } break; } + case StructuredCloneFile::eStructuredClone: + MOZ_ALWAYS_TRUE(aActors.AppendElement(null_t(), fallible)); + + break; + default: MOZ_CRASH("Should never get here!"); } } return NS_OK; } @@ -19087,30 +19112,24 @@ DatabaseOperationBase::GetStructuredClon FileManager* aFileManager, StructuredCloneReadInfo* aInfo) { MOZ_ASSERT(!IsOnBackgroundThread()); MOZ_ASSERT(aSource); MOZ_ASSERT(aFileManager); MOZ_ASSERT(aInfo); -#ifdef DEBUG - { - int32_t columnType; - MOZ_ASSERT(NS_SUCCEEDED(aSource->GetTypeOfIndex(aDataIndex, &columnType))); - MOZ_ASSERT(columnType == mozIStorageStatement::VALUE_TYPE_BLOB); - } -#endif - - const uint8_t* blobData; - uint32_t blobDataLength; - nsresult rv = aSource->GetSharedBlob(aDataIndex, &blobDataLength, &blobData); - if (NS_WARN_IF(NS_FAILED(rv))) { - return rv; - } + int32_t columnType; + nsresult rv = aSource->GetTypeOfIndex(aDataIndex, &columnType); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + MOZ_ASSERT(columnType == mozIStorageStatement::VALUE_TYPE_BLOB || + columnType == mozIStorageStatement::VALUE_TYPE_INTEGER); bool isNull; rv = aSource->GetIsNull(aFileIdsIndex, &isNull); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } nsString fileIds; @@ -19119,35 +19138,56 @@ DatabaseOperationBase::GetStructuredClon fileIds.SetIsVoid(true); } else { rv = aSource->GetString(aFileIdsIndex, fileIds); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } } - rv = GetStructuredCloneReadInfoFromBlob(blobData, - blobDataLength, - fileIds, - aFileManager, - aInfo); + if (columnType == mozIStorageStatement::VALUE_TYPE_INTEGER) { + uint64_t intData; + rv = aSource->GetInt64(aDataIndex, reinterpret_cast<int64_t*>(&intData)); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + rv = GetStructuredCloneReadInfoFromExternalBlob(intData, + aFileManager, + fileIds, + aInfo); + } else { + const uint8_t* blobData; + uint32_t blobDataLength; + nsresult rv = + aSource->GetSharedBlob(aDataIndex, &blobDataLength, &blobData); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + rv = GetStructuredCloneReadInfoFromBlob(blobData, + blobDataLength, + aFileManager, + fileIds, + aInfo); + } if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } return NS_OK; } // static nsresult DatabaseOperationBase::GetStructuredCloneReadInfoFromBlob( const uint8_t* aBlobData, uint32_t aBlobDataLength, + FileManager* aFileManager, const nsAString& aFileIds, - FileManager* aFileManager, StructuredCloneReadInfo* aInfo) { MOZ_ASSERT(!IsOnBackgroundThread()); MOZ_ASSERT(aFileManager); MOZ_ASSERT(aInfo); PROFILER_LABEL("IndexedDB", "DatabaseOperationBase::GetStructuredCloneReadInfoFromBlob", @@ -19186,16 +19226,88 @@ DatabaseOperationBase::GetStructuredClon } } return NS_OK; } // static nsresult +DatabaseOperationBase::GetStructuredCloneReadInfoFromExternalBlob( + uint64_t aIntData, + FileManager* aFileManager, + const nsAString& aFileIds, + StructuredCloneReadInfo* aInfo) +{ + MOZ_ASSERT(!IsOnBackgroundThread()); + MOZ_ASSERT(aFileManager); + MOZ_ASSERT(aInfo); + + PROFILER_LABEL( + "IndexedDB", + "DatabaseOperationBase::GetStructuredCloneReadInfoFromExternalBlob", + js::ProfileEntry::Category::STORAGE); + + nsresult rv; + + if (!aFileIds.IsVoid()) { + rv = DeserializeStructuredCloneFiles(aFileManager, aFileIds, aInfo->mFiles); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + } + + // Higher and lower 32 bits described + // in ObjectStoreAddOrPutRequestOp::DoDatabaseWork. + uint32_t index = uint32_t(aIntData); + + if (index >= aInfo->mFiles.Length()) { + MOZ_ASSERT(false, "Bad index value!"); + return NS_ERROR_UNEXPECTED; + } + + StructuredCloneFile& file = aInfo->mFiles[index]; + MOZ_ASSERT(file.mFileInfo); + MOZ_ASSERT(file.mType == StructuredCloneFile::eStructuredClone); + + nsCOMPtr<nsIFile> nativeFile = GetFileForFileInfo(file.mFileInfo); + if (NS_WARN_IF(!nativeFile)) { + return NS_ERROR_FAILURE; + } + + nsCOMPtr<nsIInputStream> stream; + rv = NS_NewLocalFileInputStream(getter_AddRefs(stream), nativeFile); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + do { + char buffer[kFileCopyBufferSize]; + + uint32_t numRead; + rv = stream->Read(buffer, sizeof(buffer), &numRead); + if (NS_WARN_IF(NS_FAILED(rv))) { + break; + } + + if (!numRead) { + break; + } + + if (NS_WARN_IF(!aInfo->mData.WriteBytes(buffer, numRead))) { + rv = NS_ERROR_OUT_OF_MEMORY; + break; + } + } while (true); + + return rv; +} + +// static +nsresult DatabaseOperationBase::BindKeyRangeToStatement( const SerializedKeyRange& aKeyRange, mozIStorageStatement* aStatement) { MOZ_ASSERT(!IsOnBackgroundThread()); MOZ_ASSERT(aStatement); nsresult rv = NS_OK; @@ -24377,17 +24489,18 @@ UpdateIndexDataValuesFunction::OnFunctio MOZ_ASSERT(valueType == mozIStorageValueArray::VALUE_TYPE_NULL || valueType == mozIStorageValueArray::VALUE_TYPE_BLOB); MOZ_ALWAYS_SUCCEEDS(aValues->GetTypeOfIndex(2, &valueType)); MOZ_ASSERT(valueType == mozIStorageValueArray::VALUE_TYPE_NULL || valueType == mozIStorageValueArray::VALUE_TYPE_TEXT); MOZ_ALWAYS_SUCCEEDS(aValues->GetTypeOfIndex(3, &valueType)); - MOZ_ASSERT(valueType == mozIStorageValueArray::VALUE_TYPE_BLOB); + MOZ_ASSERT(valueType == mozIStorageValueArray::VALUE_TYPE_BLOB || + valueType == mozIStorageValueArray::VALUE_TYPE_INTEGER); } #endif StructuredCloneReadInfo cloneInfo; nsresult rv = GetStructuredCloneReadInfoFromValueArray(aValues, /* aDataIndex */ 3, /* aFileIdsIndex */ 2, @@ -25211,16 +25324,20 @@ ObjectStoreAddOrPutRequestOp::ObjectStor MOZ_ASSERT(aParams.type() == RequestParams::TObjectStoreAddParams || aParams.type() == RequestParams::TObjectStorePutParams); mMetadata = aTransaction->GetMetadataForObjectStoreId(mParams.objectStoreId()); MOZ_ASSERT(mMetadata); mObjectStoreMayHaveIndexes = mMetadata->HasLiveIndexes(); + + mDataOverThreshold = + snappy::MaxCompressedLength(mParams.cloneInfo().data().data.Size()) > + IndexedDatabaseManager::DataThreshold(); } nsresult ObjectStoreAddOrPutRequestOp::RemoveOldIndexDataValues( DatabaseConnection* aConnection) { AssertIsOnConnectionThread(); MOZ_ASSERT(aConnection); @@ -25443,16 +25560,33 @@ ObjectStoreAddOrPutRequestOp::Init(Trans case DatabaseOrMutableFile::TPBackgroundIDBDatabaseFileChild: default: MOZ_CRASH("Should never get here!"); } } } + if (mDataOverThreshold) { + StoredFileInfo* storedFileInfo = mStoredFileInfos.AppendElement(fallible); + MOZ_ASSERT(storedFileInfo); + + if (!mFileManager) { + mFileManager = aTransaction->GetDatabase()->GetFileManager(); + MOZ_ASSERT(mFileManager); + } + + storedFileInfo->mFileInfo = mFileManager->GetNewFileInfo(); + + storedFileInfo->mInputStream = + new SCInputStream(mParams.cloneInfo().data().data); + + storedFileInfo->mType = StoredFileInfo::eStructuredClone; + } + return true; } nsresult ObjectStoreAddOrPutRequestOp::DoDatabaseWork(DatabaseConnection* aConnection) { MOZ_ASSERT(aConnection); aConnection->AssertIsOnConnectionThread(); @@ -25581,39 +25715,50 @@ ObjectStoreAddOrPutRequestOp::DoDatabase result = iter.AdvanceAcrossSegments(cloneData, 1); MOZ_ASSERT(result); } } } key.BindToStatement(stmt, NS_LITERAL_CSTRING("key")); - nsCString flatCloneData; - flatCloneData.SetLength(cloneDataSize); - auto iter = cloneData.Iter(); - cloneData.ReadBytes(iter, flatCloneData.BeginWriting(), cloneDataSize); - - // Compress the bytes before adding into the database. - const char* uncompressed = flatCloneData.BeginReading(); - size_t uncompressedLength = cloneDataSize; - - // We don't have a smart pointer class that calls free, so we need to - // manage | compressed | manually. - { + if (mDataOverThreshold) { + // Higher 32 bits reserved for flags like compressed/uncompressed. For now, + // we don't compress externally stored structured clone data since snappy + // doesn't support streaming yet and we don't want to allocate another + // large memory buffer for that. + // Lower 32 bits used for storing file_ids index. + uint64_t data = uint64_t(mStoredFileInfos.Length() - 1); + + rv = stmt->BindInt64ByName(NS_LITERAL_CSTRING("data"), data); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + } else { + nsCString flatCloneData; + flatCloneData.SetLength(cloneDataSize); + auto iter = cloneData.Iter(); + cloneData.ReadBytes(iter, flatCloneData.BeginWriting(), cloneDataSize); + + // Compress the bytes before adding into the database. + const char* uncompressed = flatCloneData.BeginReading(); + size_t uncompressedLength = cloneDataSize; + size_t compressedLength = snappy::MaxCompressedLength(uncompressedLength); - char* compressed = static_cast<char*>(malloc(compressedLength)); + UniqueFreePtr<char> compressed( + static_cast<char*>(malloc(compressedLength))); if (NS_WARN_IF(!compressed)) { return NS_ERROR_OUT_OF_MEMORY; } - snappy::RawCompress(uncompressed, uncompressedLength, compressed, + snappy::RawCompress(uncompressed, uncompressedLength, compressed.get(), &compressedLength); - uint8_t* dataBuffer = reinterpret_cast<uint8_t*>(compressed); + uint8_t* dataBuffer = reinterpret_cast<uint8_t*>(compressed.release()); size_t dataBufferLength = compressedLength; rv = stmt->BindAdoptedBlobByName(NS_LITERAL_CSTRING("data"), dataBuffer, dataBufferLength); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } } @@ -25869,20 +26014,21 @@ ObjectStoreAddOrPutRequestOp::Cleanup() { AssertIsOnOwningThread(); if (!mStoredFileInfos.IsEmpty()) { for (uint32_t count = mStoredFileInfos.Length(), index = 0; index < count; index++) { StoredFileInfo& storedFileInfo = mStoredFileInfos[index]; + + MOZ_ASSERT_IF(storedFileInfo.mType == StoredFileInfo::eMutableFile, + !storedFileInfo.mCopiedSuccessfully); + RefPtr<DatabaseFile>& fileActor = storedFileInfo.mFileActor; - - MOZ_ASSERT_IF(!fileActor, !storedFileInfo.mCopiedSuccessfully); - if (fileActor && storedFileInfo.mCopiedSuccessfully) { fileActor->ClearInputStream(); } } mStoredFileInfos.Clear(); }
--- a/dom/indexedDB/IndexedDatabase.h +++ b/dom/indexedDB/IndexedDatabase.h @@ -23,16 +23,17 @@ namespace indexedDB { class FileInfo; class SerializedStructuredCloneReadInfo; struct StructuredCloneFile { enum Type { eBlob, eMutableFile, + eStructuredClone, }; RefPtr<Blob> mBlob; RefPtr<IDBMutableFile> mMutableFile; RefPtr<FileInfo> mFileInfo; Type mType; // In IndexedDatabaseInlines.h
--- a/dom/indexedDB/PBackgroundIDBSharedTypes.ipdlh +++ b/dom/indexedDB/PBackgroundIDBSharedTypes.ipdlh @@ -49,16 +49,17 @@ struct SerializedKeyRange union NullableMutableFile { null_t; PBackgroundMutableFile; }; union BlobOrMutableFile { + null_t; PBlob; NullableMutableFile; }; struct SerializedStructuredCloneReadInfo { SerializedStructuredCloneBuffer data; BlobOrMutableFile[] blobs;