author | Grisha Kruglov <gkruglov@mozilla.com> |
Tue, 01 Nov 2016 18:55:15 -0700 | |
changeset 344864 | 4ecaf3862bc7d0feb4281ae5ec24b8495446f0c7 |
parent 344863 | 75035bf6dd1e705a75b8360677b26419e2b5b245 |
child 344865 | 63957df45bace6be17728d614625e6b358c0cc08 |
push id | 37970 |
push user | gkruglov@mozilla.com |
push date | Sat, 25 Feb 2017 01:09:28 +0000 |
treeherder | autoland@bd232d46a396 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | rnewman |
bugs | 1291821 |
milestone | 54.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/Server11RepositorySession.java +++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/Server11RepositorySession.java @@ -1,14 +1,16 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ package org.mozilla.gecko.sync.repositories; +import android.net.Uri; + import org.mozilla.gecko.background.common.log.Logger; import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionFetchRecordsDelegate; import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionGuidsSinceDelegate; import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionStoreDelegate; import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionWipeDelegate; import org.mozilla.gecko.sync.repositories.domain.Record; import org.mozilla.gecko.sync.repositories.downloaders.BatchingDownloader; import org.mozilla.gecko.sync.repositories.uploaders.BatchingUploader; @@ -30,17 +32,20 @@ public class Server11RepositorySession e return serverRepository; } @Override public void setStoreDelegate(RepositorySessionStoreDelegate storeDelegate) { super.setStoreDelegate(storeDelegate); // Now that we have the delegate, we can initialize our uploader. - this.uploader = new BatchingUploader(this, storeWorkQueue, delegate); + this.uploader = new BatchingUploader( + this, storeWorkQueue, storeDelegate, Uri.parse(serverRepository.collectionURI.toString()), + serverRepository.getCollectionLastModified(), serverRepository.getInfoConfiguration(), + serverRepository.authHeaderProvider); } @Override public void guidsSince(long timestamp, RepositorySessionGuidsSinceDelegate delegate) { // TODO Auto-generated method stub }
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/uploaders/BatchingUploader.java +++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/uploaders/BatchingUploader.java @@ -4,27 +4,22 @@ package org.mozilla.gecko.sync.repositories.uploaders; import android.net.Uri; import android.support.annotation.VisibleForTesting; import org.mozilla.gecko.background.common.log.Logger; import org.mozilla.gecko.sync.InfoConfiguration; -import org.mozilla.gecko.sync.Server11RecordPostFailedException; -import org.mozilla.gecko.sync.net.SyncResponse; -import org.mozilla.gecko.sync.net.SyncStorageResponse; -import org.mozilla.gecko.sync.repositories.Server11RepositorySession; +import org.mozilla.gecko.sync.net.AuthHeaderProvider; +import org.mozilla.gecko.sync.repositories.RepositorySession; import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionStoreDelegate; import org.mozilla.gecko.sync.repositories.domain.Record; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; /** * Uploader which implements batching introduced in Sync 1.5. * * Batch vs payload terminology: * - batch is comprised of a series of payloads, which are all committed at the same time. @@ -77,37 +72,42 @@ public class BatchingUploader { // Accessed by the record consumer thread pool. // Will be re-created, so mark it as volatile. private volatile Payload payload; // Accessed by both the record consumer thread pool and the network worker thread(s). /* package-local */ final Uri collectionUri; /* package-local */ final RepositorySessionStoreDelegate sessionStoreDelegate; /* package-local */ @VisibleForTesting final PayloadDispatcher payloadDispatcher; - private final Server11RepositorySession repositorySession; + /* package-local */ final AuthHeaderProvider authHeaderProvider; + private final RepositorySession repositorySession; // Will be re-created, so mark it as volatile. private volatile UploaderMeta uploaderMeta; // Used to ensure we have thread-safe access to the following: // - byte and record counts in both Payload and BatchMeta objects // - buffers in the Payload object private final Object payloadLock = new Object(); - - public BatchingUploader(final Server11RepositorySession repositorySession, final Executor workQueue, final RepositorySessionStoreDelegate sessionStoreDelegate) { + public BatchingUploader( + final RepositorySession repositorySession, final Executor workQueue, + final RepositorySessionStoreDelegate sessionStoreDelegate, final Uri baseCollectionUri, + final Long localCollectionLastModified, final InfoConfiguration infoConfiguration, + final AuthHeaderProvider authHeaderProvider) { this.repositorySession = repositorySession; this.sessionStoreDelegate = sessionStoreDelegate; - this.collectionUri = Uri.parse(repositorySession.getServerRepository().collectionURI().toString()); + this.collectionUri = baseCollectionUri; + this.authHeaderProvider = authHeaderProvider; - InfoConfiguration config = repositorySession.getServerRepository().getInfoConfiguration(); - this.uploaderMeta = new UploaderMeta(payloadLock, config.maxTotalBytes, config.maxTotalRecords); - this.payload = new Payload(payloadLock, config.maxPostBytes, config.maxPostRecords); + this.uploaderMeta = new UploaderMeta( + payloadLock, infoConfiguration.maxTotalBytes, infoConfiguration.maxTotalRecords); + this.payload = new Payload( + payloadLock, infoConfiguration.maxPostBytes, infoConfiguration.maxPostRecords); - this.payloadDispatcher = new PayloadDispatcher( - workQueue, this, repositorySession.getServerRepository().getCollectionLastModified()); + this.payloadDispatcher = new PayloadDispatcher(workQueue, this, localCollectionLastModified); } // Called concurrently from the threads running off of a record consumer thread pool. public void process(final Record record) { final String guid = record.guid; final byte[] recordBytes = record.toJSONBytes(); final long recordDeltaByteCount = recordBytes.length + PER_RECORD_OVERHEAD_BYTE_COUNT; @@ -192,20 +192,16 @@ public class BatchingUploader { // Will be called from a thread dispatched by PayloadDispatcher. // NB: Access to `uploaderMeta.isUnlimited` is guarded by the payloadLock. /* package-local */ void setUnlimitedMode(boolean isUnlimited) { // If we know for sure that we're not in a batching mode, // consider our batch to be of unlimited size. this.uploaderMeta.setIsUnlimited(isUnlimited); } - /* package-local */ Server11RepositorySession getRepositorySession() { - return repositorySession; - } - private void flush(final boolean isCommit, final boolean isLastPayload) { final ArrayList<byte[]> outgoing; final ArrayList<String> outgoingGuids; final long byteCount; // Even though payload object itself is thread-safe, we want to ensure we get these altogether // as a "unit". Another approach would be to create a wrapper object for these values, but this works. synchronized (payloadLock) {
--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/repositories/uploaders/BatchingUploaderTest.java +++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/repositories/uploaders/BatchingUploaderTest.java @@ -1,13 +1,14 @@ /* Any copyright is dedicated to the Public Domain. http://creativecommons.org/publicdomain/zero/1.0/ */ package org.mozilla.gecko.sync.repositories.uploaders; +import android.net.Uri; import android.support.annotation.NonNull; import static org.junit.Assert.*; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mozilla.gecko.background.testhelpers.MockRecord; @@ -423,21 +424,30 @@ public class BatchingUploaderTest { assertEquals(1, ((MockExecutorService) workQueue).commitPayloads); } private BatchingUploader makeConstrainedUploader(long maxPostRecords, long maxTotalRecords) { return makeConstrainedUploader(maxPostRecords, maxTotalRecords, false); } private BatchingUploader makeConstrainedUploader(long maxPostRecords, long maxTotalRecords, boolean firstSync) { + ExtendedJSONObject infoConfigurationJSON = new ExtendedJSONObject(); + infoConfigurationJSON.put(InfoConfiguration.MAX_TOTAL_BYTES, 4096L); + infoConfigurationJSON.put(InfoConfiguration.MAX_TOTAL_RECORDS, maxTotalRecords); + infoConfigurationJSON.put(InfoConfiguration.MAX_POST_RECORDS, maxPostRecords); + infoConfigurationJSON.put(InfoConfiguration.MAX_POST_BYTES, 1024L); + infoConfigurationJSON.put(InfoConfiguration.MAX_REQUEST_BYTES, 1024L); + Server11RepositorySession server11RepositorySession = new Server11RepositorySession( makeCountConstrainedRepository(maxPostRecords, maxTotalRecords, firstSync) ); server11RepositorySession.setStoreDelegate(storeDelegate); - return new BatchingUploader(server11RepositorySession, workQueue, storeDelegate); + return new BatchingUploader( + server11RepositorySession, workQueue, storeDelegate, Uri.EMPTY, null, + new InfoConfiguration(infoConfigurationJSON), null); } private Server11Repository makeCountConstrainedRepository(long maxPostRecords, long maxTotalRecords, boolean firstSync) { return makeConstrainedRepository(1024, 1024, maxPostRecords, 4096, maxTotalRecords, firstSync); } private Server11Repository makeConstrainedRepository(long maxRequestBytes, long maxPostBytes, long maxPostRecords, long maxTotalBytes, long maxTotalRecords, boolean firstSync) { ExtendedJSONObject infoConfigurationJSON = new ExtendedJSONObject(); @@ -461,16 +471,17 @@ public class BatchingUploaderTest { infoCollections = new InfoCollections() { @Override public Long getTimestamp(String collection) { return 0L; } }; } + try { return new Server11Repository( "dummyCollection", "http://dummy.url/", null, infoCollections, infoConfiguration );
--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/repositories/uploaders/PayloadUploadDelegateTest.java +++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/repositories/uploaders/PayloadUploadDelegateTest.java @@ -3,41 +3,34 @@ package org.mozilla.gecko.sync.repositories.uploaders; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mozilla.gecko.background.testhelpers.TestRunner; import org.mozilla.gecko.sync.HTTPFailureException; -import org.mozilla.gecko.sync.InfoCollections; -import org.mozilla.gecko.sync.InfoConfiguration; import org.mozilla.gecko.sync.NonObjectJSONException; import org.mozilla.gecko.sync.net.AuthHeaderProvider; import org.mozilla.gecko.sync.net.SyncResponse; import org.mozilla.gecko.sync.net.SyncStorageResponse; -import org.mozilla.gecko.sync.repositories.RepositorySession; -import org.mozilla.gecko.sync.repositories.Server11Repository; -import org.mozilla.gecko.sync.repositories.Server11RepositorySession; import static org.mockito.Mockito.mock; import java.io.ByteArrayInputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.concurrent.Executor; import ch.boye.httpclientandroidlib.HttpResponse; import ch.boye.httpclientandroidlib.ProtocolVersion; import ch.boye.httpclientandroidlib.entity.BasicHttpEntity; import ch.boye.httpclientandroidlib.message.BasicHttpResponse; import ch.boye.httpclientandroidlib.message.BasicStatusLine; -import static org.mockito.Mockito.mock; - import static org.junit.Assert.*; @RunWith(TestRunner.class) public class PayloadUploadDelegateTest { private PayloadDispatcher payloadDispatcher; private AuthHeaderProvider authHeaderProvider; class MockPayloadDispatcher extends PayloadDispatcher { @@ -79,28 +72,16 @@ public class PayloadUploadDelegateTest { @Override public void lastPayloadFailed() { didLastPayloadFail = true; } } @Before public void setUp() throws Exception { - Server11Repository server11Repository = new Server11Repository( - "dummyCollection", - "http://dummy.url/", - null, - new InfoCollections() { - @Override - public Long getTimestamp(String collection) { - return 0L; - } - }, - new InfoConfiguration() - ); payloadDispatcher = new MockPayloadDispatcher( null, mock(BatchingUploader.class) ); authHeaderProvider = mock(AuthHeaderProvider.class); }