Bug 1346438 - Specify X-I-U-S header value while uploading meta/global r=nalexander
authorGrigory Kruglov <gkruglov@mozilla.com>
Mon, 20 Mar 2017 16:40:13 -0700
changeset 348463 3250c8e72a981a64c2d675cd305791e27e431bbf
parent 348462 737be6c1576e6a5af463d41b3dbb7fa7c1e739e0
child 348464 0db18c13ea0ab3958dc8ce525455912cfb79d8a7
push id39190
push usergkruglov@mozilla.com
push dateMon, 20 Mar 2017 23:53:20 +0000
treeherderautoland@0db18c13ea0a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1346438
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1346438 - Specify X-I-U-S header value while uploading meta/global r=nalexander We upload meta/global in three scenarios: - fresh start - when it was modified after a successful sync - when it was modified after an aborted sync Use X-I-U-S header to assert what we believe about meta/global's presence (during freshStart) and last-modified timestamp (in all other cases). We might encounter a concurrent modification condition, manifesting as a 412 error. If we see such an error: - on fresh start, we restart globalSession - on regular upload, we request a re-sync of all stages MozReview-Commit-ID: 3qyb6rUSOeY
mobile/android/services/src/main/java/org/mozilla/gecko/fxa/sync/FxAccountSyncAdapter.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/GlobalSession.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/InfoCollections.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/MetaGlobal.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/delegates/GlobalSessionCallback.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FetchMetaGlobalStage.java
mobile/android/tests/background/junit3/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestGlobalSession.java
mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestMetaGlobal.java
mobile/android/tests/background/junit4/src/org/mozilla/android/sync/test/helpers/MockGlobalSessionCallback.java
mobile/android/tests/background/junit4/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/stage/test/TestFetchMetaGlobalStage.java
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/fxa/sync/FxAccountSyncAdapter.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/fxa/sync/FxAccountSyncAdapter.java
@@ -29,16 +29,17 @@ import org.mozilla.gecko.fxa.authenticat
 import org.mozilla.gecko.fxa.authenticator.FxAccountAuthenticator;
 import org.mozilla.gecko.fxa.login.FxAccountLoginStateMachine;
 import org.mozilla.gecko.fxa.login.Married;
 import org.mozilla.gecko.fxa.login.State;
 import org.mozilla.gecko.fxa.login.State.StateLabel;
 import org.mozilla.gecko.fxa.sync.FxAccountSyncDelegate.Result;
 import org.mozilla.gecko.sync.BackoffHandler;
 import org.mozilla.gecko.sync.GlobalSession;
+import org.mozilla.gecko.sync.MetaGlobal;
 import org.mozilla.gecko.sync.PrefsBackoffHandler;
 import org.mozilla.gecko.sync.SharedPreferencesClientsDataDelegate;
 import org.mozilla.gecko.sync.SyncConfiguration;
 import org.mozilla.gecko.sync.ThreadPool;
 import org.mozilla.gecko.sync.Utils;
 import org.mozilla.gecko.sync.crypto.KeyBundle;
 import org.mozilla.gecko.sync.delegates.GlobalSessionCallback;
 import org.mozilla.gecko.sync.delegates.ClientsDataDelegate;
@@ -127,16 +128,17 @@ public class FxAccountSyncAdapter extend
     /* package-local */ void requestFollowUpSync(String stage) {
       this.stageNamesForFollowUpSync.add(stage);
     }
 
     protected final Collection<String> stageNamesToSync;
 
     // Keeps track of incomplete stages during this sync that need to be re-synced once we're done.
     private final List<String> stageNamesForFollowUpSync = Collections.synchronizedList(new ArrayList<String>());
+    private boolean fullSyncNecessary = false;
 
     public SyncDelegate(BlockingQueue<Result> latch, SyncResult syncResult, AndroidFxAccount fxAccount, Collection<String> stageNamesToSync) {
       super(latch, syncResult);
       this.stageNamesToSync = Collections.unmodifiableCollection(stageNamesToSync);
     }
 
     public Collection<String> getStageNamesToSync() {
       return this.stageNamesToSync;
@@ -195,16 +197,24 @@ public class FxAccountSyncAdapter extend
      * Schedule an incomplete stage for a follow-up sync.
      */
     @Override
     public void handleIncompleteStage(Stage currentState,
                                       GlobalSession globalSession) {
       syncDelegate.requestFollowUpSync(currentState.getRepositoryName());
     }
 
+    /**
+     * Use with caution, as this will request an immediate follow-up sync of all stages.
+     */
+    @Override
+    public void handleFullSyncNecessary() {
+      syncDelegate.fullSyncNecessary = true;
+    }
+
     @Override
     public void handleSuccess(GlobalSession globalSession) {
       Logger.info(LOG_TAG, "Global session succeeded.");
 
       // Get the number of clients, so we can schedule the sync interval accordingly.
       try {
         int otherClientsCount = globalSession.getClientsDelegate().getClientsCount();
         Logger.debug(LOG_TAG, "" + otherClientsCount + " other client(s).");
@@ -449,16 +459,17 @@ public class FxAccountSyncAdapter extend
     });
 
     final BlockingQueue<Result> latch = new LinkedBlockingQueue<>(1);
 
     Collection<String> knownStageNames = SyncConfiguration.validEngineNames();
     Collection<String> stageNamesToSync = Utils.getStagesToSyncFromBundle(knownStageNames, extras);
 
     final SyncDelegate syncDelegate = new SyncDelegate(latch, syncResult, fxAccount, stageNamesToSync);
+    Result offeredResult = null;
 
     try {
       // This will be the same chunk of SharedPreferences that we pass through to GlobalSession/SyncConfiguration.
       final SharedPreferences sharedPrefs = fxAccount.getSyncPrefs();
 
       final BackoffHandler backgroundBackoffHandler = new PrefsBackoffHandler(sharedPrefs, "background");
       final BackoffHandler rateLimitBackoffHandler = new PrefsBackoffHandler(sharedPrefs, "rate");
 
@@ -586,37 +597,57 @@ public class FxAccountSyncAdapter extend
             fxAccount.fetchProfileJSON();
           } catch (Exception e) {
             syncDelegate.handleError(e);
             return;
           }
         }
       });
 
-      latch.take();
+      offeredResult = latch.take();
     } catch (Exception e) {
       Logger.error(LOG_TAG, "Got error syncing.", e);
       syncDelegate.handleError(e);
     } finally {
       fxAccount.releaseSharedAccountStateLock();
     }
 
+    lastSyncRealtimeMillis = SystemClock.elapsedRealtime();
+
+    // We got to this point without being offered a result, and so it's unwise to proceed with
+    // trying to sync stages again. Nothing else we can do but log an error.
+    if (offeredResult == null) {
+      Logger.error(LOG_TAG, "Did not receive a sync result from the delegate.");
+      return;
+    }
+
+    // Full sync (of all of stages) is necessary if we hit "concurrent modification" errors while
+    // uploading meta/global stage. This is considered both a rare and important event, so it's
+    // deemed safe and necessary to request an immediate sync, which will ignore any back-offs and
+    // will happen right away.
+    if (syncDelegate.fullSyncNecessary) {
+      Logger.info(LOG_TAG, "Syncing done. Full follow-up sync necessary, requesting immediate sync.");
+      fxAccount.requestImmediateSync(null, null);
+      return;
+    }
+
     // If there are any incomplete stages, request a follow-up sync. Otherwise, we're done.
     // Incomplete stage is:
     // - one that hit a 412 error during either upload or download of data, indicating that
     //   its collection has been modified remotely, or
     // - one that hit a sync deadline
     final String[] stagesToSyncAgain;
     synchronized (syncDelegate.stageNamesForFollowUpSync) {
       stagesToSyncAgain = syncDelegate.stageNamesForFollowUpSync.toArray(
               new String[syncDelegate.stageNamesForFollowUpSync.size()]
       );
     }
 
-    if (stagesToSyncAgain.length > 0) {
-      Logger.info(LOG_TAG, "Syncing done. Requesting an immediate follow-up sync.");
-      fxAccount.requestImmediateSync(stagesToSyncAgain, null);
-    } else {
+    if (stagesToSyncAgain.length == 0) {
       Logger.info(LOG_TAG, "Syncing done.");
+      return;
     }
-    lastSyncRealtimeMillis = SystemClock.elapsedRealtime();
+
+    // If there are any other stages marked as incomplete, request that they're synced again.
+    Logger.info(LOG_TAG, "Syncing done. Requesting an immediate follow-up sync.");
+    fxAccount.requestImmediateSync(stagesToSyncAgain, null);
   }
 }
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/GlobalSession.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/GlobalSession.java
@@ -1,15 +1,16 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.sync;
 
 import android.content.Context;
+import android.support.annotation.VisibleForTesting;
 
 import org.json.simple.JSONArray;
 import org.mozilla.gecko.background.common.log.Logger;
 import org.mozilla.gecko.sync.crypto.CryptoException;
 import org.mozilla.gecko.sync.crypto.KeyBundle;
 import org.mozilla.gecko.sync.delegates.ClientsDataDelegate;
 import org.mozilla.gecko.sync.delegates.FreshStartDelegate;
 import org.mozilla.gecko.sync.delegates.GlobalSessionCallback;
@@ -404,69 +405,91 @@ public class GlobalSession implements Ht
     updateMetaGlobalInPlace();
 
     Logger.debug(LOG_TAG, "Uploading updated meta/global record.");
     final Object monitor = new Object();
 
     Runnable doUpload = new Runnable() {
       @Override
       public void run() {
-        config.metaGlobal.upload(new MetaGlobalDelegate() {
-          @Override
-          public void handleSuccess(MetaGlobal global, SyncStorageResponse response) {
-            Logger.info(LOG_TAG, "Successfully uploaded updated meta/global record.");
-            // Engine changes are stored as diffs, so update enabled engines in config to match uploaded meta/global.
-            config.enabledEngineNames = config.metaGlobal.getEnabledEngineNames();
-            // Clear userSelectedEngines because they are updated in config and meta/global.
-            config.userSelectedEngines = null;
-
-            synchronized (monitor) {
-              monitor.notify();
-            }
-          }
-
-          @Override
-          public void handleMissing(MetaGlobal global, SyncStorageResponse response) {
-            Logger.warn(LOG_TAG, "Got 404 missing uploading updated meta/global record; shouldn't happen.  Ignoring.");
-            synchronized (monitor) {
-              monitor.notify();
-            }
-          }
-
-          @Override
-          public void handleFailure(SyncStorageResponse response) {
-            Logger.warn(LOG_TAG, "Failed to upload updated meta/global record; ignoring.");
-            synchronized (monitor) {
-              monitor.notify();
-            }
-          }
-
-          @Override
-          public void handleError(Exception e) {
-            Logger.warn(LOG_TAG, "Got exception trying to upload updated meta/global record; ignoring.", e);
-            synchronized (monitor) {
-              monitor.notify();
-            }
-          }
-        });
+        // During regular meta/global upload, set X-I-U-S to the last-modified value of meta/global
+        // in info/collections, to ensure we catch concurrent modifications by other clients.
+        Long lastModifiedTimestamp = config.infoCollections.getTimestamp("meta");
+        // Theoretically, meta/global's timestamp might be missing from info/collections.
+        // The safest thing in that case is to assert that meta/global hasn't been modified by other
+        // clients by setting X-I-U-S to 0.
+        // See Bug 1346438.
+        if (lastModifiedTimestamp == null) {
+          lastModifiedTimestamp = 0L;
+        }
+        config.metaGlobal.upload(lastModifiedTimestamp, makeMetaGlobalUploadDelegate(config, callback, monitor));
       }
     };
 
     final Thread upload = new Thread(doUpload);
     synchronized (monitor) {
       try {
         upload.start();
         monitor.wait();
         Logger.debug(LOG_TAG, "Uploaded updated meta/global record.");
       } catch (InterruptedException e) {
         Logger.error(LOG_TAG, "Uploading updated meta/global interrupted; continuing.");
       }
     }
   }
 
+  @VisibleForTesting
+  public static MetaGlobalDelegate makeMetaGlobalUploadDelegate(final SyncConfiguration config, final GlobalSessionCallback callback, final Object monitor) {
+    return new MetaGlobalDelegate() {
+      @Override
+      public void handleSuccess(MetaGlobal global, SyncStorageResponse response) {
+        Logger.info(LOG_TAG, "Successfully uploaded updated meta/global record.");
+        // Engine changes are stored as diffs, so update enabled engines in config to match uploaded meta/global.
+        config.enabledEngineNames = config.metaGlobal.getEnabledEngineNames();
+        // Clear userSelectedEngines because they are updated in config and meta/global.
+        config.userSelectedEngines = null;
+
+        synchronized (monitor) {
+          monitor.notify();
+        }
+      }
+
+      @Override
+      public void handleMissing(MetaGlobal global, SyncStorageResponse response) {
+        Logger.warn(LOG_TAG, "Got 404 missing uploading updated meta/global record; shouldn't happen.  Ignoring.");
+        synchronized (monitor) {
+          monitor.notify();
+        }
+      }
+
+      @Override
+      public void handleFailure(SyncStorageResponse response) {
+        Logger.warn(LOG_TAG, "Failed to upload updated meta/global record; ignoring.");
+
+        // If we encountered a concurrent modification while uploading meta/global, request that
+        // sync of all stages happens once we're done.
+        if (response.getStatusCode() == 412) {
+          callback.handleFullSyncNecessary();
+        }
+
+        synchronized (monitor) {
+          monitor.notify();
+        }
+      }
+
+      @Override
+      public void handleError(Exception e) {
+        Logger.warn(LOG_TAG, "Got exception trying to upload updated meta/global record; ignoring.", e);
+        synchronized (monitor) {
+          monitor.notify();
+        }
+      }
+    };
+  }
+
 
   public void abort(Exception e, String reason) {
     Logger.warn(LOG_TAG, "Aborting sync: " + reason, e);
     cleanUp();
     long existingBackoff = largestBackoffObserved.get();
     if (existingBackoff > 0) {
       callback.requestBackoff(existingBackoff);
     }
@@ -704,36 +727,56 @@ public class GlobalSession implements Ht
   public void processMissingMetaGlobal(MetaGlobal global) {
     freshStart();
   }
 
   /**
    * Do a fresh start then quietly finish the sync, starting another.
    */
   public void freshStart() {
-    final GlobalSession globalSession = this;
-    freshStart(this, new FreshStartDelegate() {
+    freshStart(this, makeFreshStartDelegate(this));
+  }
 
+  @VisibleForTesting
+  public static FreshStartDelegate makeFreshStartDelegate(final GlobalSession globalSession) {
+    return new FreshStartDelegate() {
       @Override
       public void onFreshStartFailed(Exception e) {
-        globalSession.abort(e, "Fresh start failed.");
+        if (!(e instanceof  HTTPFailureException)) {
+          globalSession.abort(e, "Fresh start failed.");
+          return;
+        }
+
+        if (((HTTPFailureException) e).response.getStatusCode() != 412) {
+          globalSession.abort(e, "Fresh start failed with non-412 status code.");
+          return;
+        }
+
+        // In case of a concurrent modification during a fresh start, restart global session.
+        try {
+          // We are not persisting SyncConfiguration at this point; we can't be sure of its state.
+          globalSession.restart();
+        } catch (AlreadySyncingException restartException) {
+          Logger.warn(LOG_TAG, "Got exception restarting sync after freshStart failure.", restartException);
+          globalSession.abort(restartException, "Got exception restarting sync after freshStart failure.");
+        }
       }
 
       @Override
       public void onFreshStart() {
         try {
           Logger.warn(LOG_TAG, "Fresh start succeeded; restarting global session.");
           globalSession.config.persistToPrefs();
           globalSession.restart();
         } catch (Exception e) {
           Logger.warn(LOG_TAG, "Got exception when restarting sync after freshStart.", e);
           globalSession.abort(e, "Got exception after freshStart.");
         }
       }
-    });
+    };
   }
 
   /**
    * Clean the server, aborting the current sync.
    * <p>
    * <ol>
    * <li>Wipe the server storage.</li>
    * <li>Reset all stages and purge cached state: (meta/global and crypto/keys records).</li>
@@ -757,21 +800,21 @@ public class GlobalSession implements Ht
 
         session.resetAllStages();
         session.config.purgeMetaGlobal();
         session.config.purgeCryptoKeys();
         session.config.persistToPrefs();
 
         Logger.info(LOG_TAG, "Uploading new meta/global with sync ID " + mg.syncID + ".");
 
-        // It would be good to set the X-If-Unmodified-Since header to `timestamp`
-        // for this PUT to ensure at least some level of transactionality.
-        // Unfortunately, the servers don't support it after a wipe right now
-        // (bug 693893), so we're going to defer this until bug 692700.
-        mg.upload(new MetaGlobalDelegate() {
+        // During a fresh start, set X-I-U-S to 0 to ensure we don't race with other clients.
+        // Since we are performing a fresh start, we are asserting that meta/global was not uploaded
+        // by other clients.
+        // See Bug 1346438.
+        mg.upload(0L, new MetaGlobalDelegate() {
           @Override
           public void handleSuccess(MetaGlobal uploadedGlobal, SyncStorageResponse uploadResponse) {
             Logger.info(LOG_TAG, "Uploaded new meta/global with sync ID " + uploadedGlobal.syncID + ".");
 
             // Generate new keys.
             CollectionKeys keys = null;
             try {
               keys = session.generateNewCryptoKeys();
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/InfoCollections.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/InfoCollections.java
@@ -1,14 +1,16 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.sync;
 
+import android.support.annotation.Nullable;
+
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 
 import org.mozilla.gecko.background.common.log.Logger;
 
 /**
@@ -62,16 +64,17 @@ public class InfoCollections {
   /**
    * Return the timestamp for the given collection, or null if the timestamps
    * have not been fetched or the given collection does not have a timestamp.
    *
    * @param collection
    *          The collection to inspect.
    * @return the timestamp in milliseconds since epoch.
    */
+  @Nullable
   public Long getTimestamp(String collection) {
     if (timestamps == null) {
       return null;
     }
     return timestamps.get(collection);
   }
 
   /**
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/MetaGlobal.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/MetaGlobal.java
@@ -17,17 +17,17 @@ import org.mozilla.gecko.background.comm
 import org.mozilla.gecko.sync.MetaGlobalException.MetaGlobalMalformedSyncIDException;
 import org.mozilla.gecko.sync.MetaGlobalException.MetaGlobalMalformedVersionException;
 import org.mozilla.gecko.sync.delegates.MetaGlobalDelegate;
 import org.mozilla.gecko.sync.net.AuthHeaderProvider;
 import org.mozilla.gecko.sync.net.SyncStorageRecordRequest;
 import org.mozilla.gecko.sync.net.SyncStorageRequestDelegate;
 import org.mozilla.gecko.sync.net.SyncStorageResponse;
 
-public class MetaGlobal implements SyncStorageRequestDelegate {
+public class MetaGlobal {
   private static final String LOG_TAG = "MetaGlobal";
   protected String metaURL;
 
   // Fields.
   protected ExtendedJSONObject  engines;
   protected JSONArray           declined;
   protected Long                storageVersion;
   protected String              syncID;
@@ -49,29 +49,28 @@ public class MetaGlobal implements SyncS
     this.authHeaderProvider = authHeaderProvider;
   }
 
   public void fetch(MetaGlobalDelegate delegate) {
     this.callback = delegate;
     try {
       this.isUploading = false;
       SyncStorageRecordRequest r = new SyncStorageRecordRequest(this.metaURL);
-      r.delegate = this;
+      r.delegate = new MetaUploadDelegate(this, null);
       r.get();
     } catch (URISyntaxException e) {
       this.callback.handleError(e);
     }
   }
 
-  public void upload(MetaGlobalDelegate callback) {
+  public void upload(long lastModifiedTimestamp, MetaGlobalDelegate callback) {
     try {
       this.isUploading = true;
       SyncStorageRecordRequest r = new SyncStorageRecordRequest(this.metaURL);
-
-      r.delegate = this;
+      r.delegate = new MetaUploadDelegate(this, lastModifiedTimestamp);
       this.callback = callback;
       r.put(this.asCryptoRecord());
     } catch (Exception e) {
       callback.handleError(e);
     }
   }
 
   protected ExtendedJSONObject asRecordContents() {
@@ -314,35 +313,16 @@ public class MetaGlobal implements SyncS
     this.syncID = syncID;
   }
 
   // SyncStorageRequestDelegate methods for fetching.
   public String credentials() {
     return null;
   }
 
-  @Override
-  public AuthHeaderProvider getAuthHeaderProvider() {
-    return authHeaderProvider;
-  }
-
-  @Override
-  public String ifUnmodifiedSince() {
-    return null;
-  }
-
-  @Override
-  public void handleRequestSuccess(SyncStorageResponse response) {
-    if (this.isUploading) {
-      this.handleUploadSuccess(response);
-    } else {
-      this.handleDownloadSuccess(response);
-    }
-  }
-
   private void handleUploadSuccess(SyncStorageResponse response) {
     this.callback.handleSuccess(this, response);
   }
 
   private void handleDownloadSuccess(SyncStorageResponse response) {
     if (response.wasSuccessful()) {
       try {
         CryptoRecord record = CryptoRecord.fromJSONRecord(response.jsonObjectBody());
@@ -351,22 +331,54 @@ public class MetaGlobal implements SyncS
       } catch (Exception e) {
         this.callback.handleError(e);
       }
       return;
     }
     this.callback.handleFailure(response);
   }
 
-  @Override
-  public void handleRequestFailure(SyncStorageResponse response) {
-    if (response.getStatusCode() == 404) {
-      this.callback.handleMissing(this, response);
-      return;
+  private static class MetaUploadDelegate implements SyncStorageRequestDelegate {
+    private final MetaGlobal metaGlobal;
+    private final Long ifUnmodifiedSinceTimestamp;
+
+    /* package-local */ MetaUploadDelegate(final MetaGlobal metaGlobal, final Long ifUnmodifiedSinceTimestamp) {
+      this.metaGlobal = metaGlobal;
+      this.ifUnmodifiedSinceTimestamp = ifUnmodifiedSinceTimestamp;
+    }
+
+    @Override
+    public AuthHeaderProvider getAuthHeaderProvider() {
+      return metaGlobal.authHeaderProvider;
+    }
+
+    @Override
+    public String ifUnmodifiedSince() {
+      if (ifUnmodifiedSinceTimestamp == null) {
+        return null;
+      }
+      return Utils.millisecondsToDecimalSecondsString(ifUnmodifiedSinceTimestamp);
     }
-    this.callback.handleFailure(response);
-  }
+
+    @Override
+    public void handleRequestSuccess(SyncStorageResponse response) {
+      if (metaGlobal.isUploading) {
+        metaGlobal.handleUploadSuccess(response);
+      } else {
+        metaGlobal.handleDownloadSuccess(response);
+      }
+    }
 
-  @Override
-  public void handleRequestError(Exception e) {
-    this.callback.handleError(e);
+    @Override
+    public void handleRequestFailure(SyncStorageResponse response) {
+      if (response.getStatusCode() == 404) {
+        metaGlobal.callback.handleMissing(metaGlobal, response);
+        return;
+      }
+      metaGlobal.callback.handleFailure(response);
+    }
+
+    @Override
+    public void handleRequestError(Exception e) {
+      metaGlobal.callback.handleError(e);
+    }
   }
 }
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/delegates/GlobalSessionCallback.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/delegates/GlobalSessionCallback.java
@@ -34,16 +34,17 @@ public interface GlobalSessionCallback {
    */
   void informMigrated(GlobalSession session);
 
   void handleAborted(GlobalSession globalSession, String reason);
   void handleError(GlobalSession globalSession, Exception ex);
   void handleSuccess(GlobalSession globalSession);
   void handleStageCompleted(Stage currentState, GlobalSession globalSession);
   void handleIncompleteStage(Stage currentState, GlobalSession globalSession);
+  void handleFullSyncNecessary();
 
   /**
    * Called when a {@link GlobalSession} wants to know if it should continue
    * to make storage requests.
    *
    * @return false if the session should make no further requests.
    */
   boolean shouldBackOffStorage();
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FetchMetaGlobalStage.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FetchMetaGlobalStage.java
@@ -53,27 +53,28 @@ public class FetchMetaGlobalStage extend
   @Override
   public void execute() throws NoSuchStageException {
     InfoCollections infoCollections = session.config.infoCollections;
     if (infoCollections == null) {
       session.abort(null, "No info/collections set in FetchMetaGlobalStage.");
       return;
     }
 
-    long lastModified = session.config.persistedMetaGlobal().lastModified();
+    final long lastModified = session.config.persistedMetaGlobal().lastModified();
     if (!infoCollections.updateNeeded(META_COLLECTION, lastModified)) {
       // Try to use our local collection keys for this session.
       Logger.info(LOG_TAG, "Trying to use persisted meta/global for this session.");
       MetaGlobal global = session.config.persistedMetaGlobal().metaGlobal(session.config.metaURL(), session.getAuthHeaderProvider());
       if (global != null) {
         Logger.info(LOG_TAG, "Using persisted meta/global for this session.");
         session.processMetaGlobal(global); // Calls session.advance().
         return;
       }
       Logger.info(LOG_TAG, "Failed to use persisted meta/global for this session.");
     }
 
     // We need an update: fetch or upload meta/global as necessary.
+    // We assert when we believe meta/global was last modified via X-I-U-S.
     Logger.info(LOG_TAG, "Fetching fresh meta/global for this session.");
     MetaGlobal global = new MetaGlobal(session.config.metaURL(), session.getAuthHeaderProvider());
     global.fetch(new StageMetaGlobalDelegate(session));
   }
 }
--- a/mobile/android/tests/background/junit3/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
+++ b/mobile/android/tests/background/junit3/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
@@ -30,16 +30,21 @@ public class DefaultGlobalSessionCallbac
 
   @Override
   public void handleIncompleteStage(Stage currentState,
                                     GlobalSession globalSession) {
 
   }
 
   @Override
+  public void handleFullSyncNecessary() {
+
+  }
+
+  @Override
   public void handleAborted(GlobalSession globalSession, String reason) {
   }
 
   @Override
   public void handleError(GlobalSession globalSession, Exception ex) {
   }
 
   @Override
--- a/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestGlobalSession.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestGlobalSession.java
@@ -22,23 +22,26 @@ import org.mozilla.gecko.background.test
 import org.mozilla.gecko.background.testhelpers.MockPrefsGlobalSession;
 import org.mozilla.gecko.background.testhelpers.MockServerSyncStage;
 import org.mozilla.gecko.background.testhelpers.MockSharedPreferences;
 import org.mozilla.gecko.background.testhelpers.TestRunner;
 import org.mozilla.gecko.background.testhelpers.WaitHelper;
 import org.mozilla.gecko.sync.EngineSettings;
 import org.mozilla.gecko.sync.ExtendedJSONObject;
 import org.mozilla.gecko.sync.GlobalSession;
+import org.mozilla.gecko.sync.HTTPFailureException;
+import org.mozilla.gecko.sync.InfoCollections;
 import org.mozilla.gecko.sync.MetaGlobal;
 import org.mozilla.gecko.sync.NonObjectJSONException;
 import org.mozilla.gecko.sync.SyncConfiguration;
 import org.mozilla.gecko.sync.SyncConfigurationException;
 import org.mozilla.gecko.sync.Utils;
 import org.mozilla.gecko.sync.crypto.CryptoException;
 import org.mozilla.gecko.sync.crypto.KeyBundle;
+import org.mozilla.gecko.sync.delegates.MetaGlobalDelegate;
 import org.mozilla.gecko.sync.net.BaseResource;
 import org.mozilla.gecko.sync.net.BasicAuthHeaderProvider;
 import org.mozilla.gecko.sync.net.SyncStorageResponse;
 import org.mozilla.gecko.sync.repositories.domain.VersionConstants;
 import org.mozilla.gecko.sync.stage.AndroidBrowserBookmarksServerSyncStage;
 import org.mozilla.gecko.sync.stage.GlobalSyncStage;
 import org.mozilla.gecko.sync.stage.GlobalSyncStage.Stage;
 import org.mozilla.gecko.sync.stage.NoSuchStageException;
@@ -54,16 +57,17 @@ import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
 
 @RunWith(TestRunner.class)
 public class TestGlobalSession {
   private int          TEST_PORT                = HTTPServerTestHelper.getTestPort();
   private final String TEST_CLUSTER_URL         = "http://localhost:" + TEST_PORT;
   private final String TEST_USERNAME            = "johndoe";
   private final String TEST_PASSWORD            = "password";
   private final String TEST_SYNC_KEY            = "abcdeabcdeabcdeabcdeabcdea";
@@ -386,16 +390,17 @@ public class TestGlobalSession {
 
   @Test
   public void testUploadUpdatedMetaGlobal() throws Exception {
     // Set up session with meta/global.
     final MockGlobalSessionCallback callback = new MockGlobalSessionCallback();
     final GlobalSession session = MockPrefsGlobalSession.getSession(TEST_USERNAME, TEST_PASSWORD,
         new KeyBundle(TEST_USERNAME, TEST_SYNC_KEY), callback, null, null);
     session.config.metaGlobal = session.generateNewMetaGlobal();
+    session.config.infoCollections = mock(InfoCollections.class);
     session.enginesToUpdate.clear();
 
     // Set enabledEngines in meta/global, including a "new engine."
     String[] origEngines = new String[] { "bookmarks", "clients", "forms", "history", "tabs", "new-engine" };
 
     ExtendedJSONObject origEnginesJSONObject = new ExtendedJSONObject();
     for (String engineName : origEngines) {
       EngineSettings mockEngineSettings = new EngineSettings(Utils.generateGuid(), Integer.valueOf(0));
@@ -428,13 +433,58 @@ public class TestGlobalSession {
       expected.remove(name);
     }
     for (String name : toAdd) {
       expected.add(name);
     }
     assertEquals(expected, session.config.metaGlobal.getEnabledEngineNames());
   }
 
+  @Test
+  public void testUploadMetaGlobalDelegate412() {
+    final Object monitor = new Object();
+    final MockGlobalSessionCallback callback = new MockGlobalSessionCallback();
+    MetaGlobalDelegate metaGlobalDelegate = GlobalSession.makeMetaGlobalUploadDelegate(
+            mock(SyncConfiguration.class),
+            callback,
+            monitor
+    );
+
+    metaGlobalDelegate.handleFailure(makeSyncStorageResponse(412));
+
+    assertTrue(callback.calledFullSyncNecessary);
+  }
+
+  @Test
+  public void testUploadMetaGlobalDelegateNon412() {
+    final Object monitor = new Object();
+    final MockGlobalSessionCallback callback = new MockGlobalSessionCallback();
+    MetaGlobalDelegate metaGlobalDelegate = GlobalSession.makeMetaGlobalUploadDelegate(
+            mock(SyncConfiguration.class),
+            callback,
+            monitor
+    );
+
+    metaGlobalDelegate.handleFailure(makeSyncStorageResponse(400));
+
+    assertFalse(callback.calledFullSyncNecessary);
+  }
+
   public void testStageAdvance() {
     assertEquals(GlobalSession.nextStage(Stage.idle), Stage.checkPreconditions);
     assertEquals(GlobalSession.nextStage(Stage.completed), Stage.idle);
   }
+
+  public static HTTPFailureException makeHttpFailureException(int statusCode) {
+    return new HTTPFailureException(makeSyncStorageResponse(statusCode));
+  }
+
+  public static SyncStorageResponse makeSyncStorageResponse(int statusCode) {
+    // \\( >.<)//
+    return new SyncStorageResponse(
+            new BasicHttpResponse(
+                    new BasicStatusLine(
+                            new ProtocolVersion("HTTP", 1, 1), statusCode, null
+                    )
+            )
+    );
+  }
 }
--- a/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestMetaGlobal.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestMetaGlobal.java
@@ -291,17 +291,17 @@ public class TestMetaGlobal {
   }
 
 
   public MockMetaGlobalFetchDelegate doUpload(final MetaGlobal global) {
     final MockMetaGlobalFetchDelegate delegate = new MockMetaGlobalFetchDelegate();
     WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(new Runnable() {
       @Override
       public void run() {
-        global.upload(delegate);
+        global.upload(0L, delegate);
       }
     }));
 
     return delegate;
   }
 
   @Test
   public void testUpload() {
--- a/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/test/helpers/MockGlobalSessionCallback.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/test/helpers/MockGlobalSessionCallback.java
@@ -4,16 +4,17 @@
 package org.mozilla.android.sync.test.helpers;
 
 import org.mozilla.gecko.background.testhelpers.WaitHelper;
 import org.mozilla.gecko.sync.GlobalSession;
 import org.mozilla.gecko.sync.delegates.GlobalSessionCallback;
 import org.mozilla.gecko.sync.stage.GlobalSyncStage.Stage;
 
 import java.net.URI;
+import java.util.ArrayList;
 
 import static org.junit.Assert.assertEquals;
 
 /**
  * A callback for use with a GlobalSession that records what happens for later
  * inspection.
  *
  * This callback is expected to be used from within the friendly confines of a
@@ -30,16 +31,18 @@ public class MockGlobalSessionCallback i
   public Exception calledErrorException = null;
   public boolean calledAborted = false;
   public boolean calledRequestBackoff = false;
   public boolean calledInformUnauthorizedResponse = false;
   public boolean calledInformUpgradeRequiredResponse = false;
   public boolean calledInformMigrated = false;
   public URI calledInformUnauthorizedResponseClusterURL = null;
   public long weaveBackoff = -1;
+  public boolean calledFullSyncNecessary = false;
+  public ArrayList<String> incompleteStages = new ArrayList<>();
 
   @Override
   public void handleSuccess(GlobalSession globalSession) {
     this.calledSuccess = true;
     assertEquals(0, this.stageCounter);
     this.testWaiter().performNotify();
   }
 
@@ -54,17 +57,22 @@ public class MockGlobalSessionCallback i
     this.calledError = true;
     this.calledErrorException = ex;
     this.testWaiter().performNotify();
   }
 
   @Override
   public void handleIncompleteStage(Stage currentState,
                                     GlobalSession globalSession) {
+    this.incompleteStages.add(currentState.getRepositoryName());
+  }
 
+  @Override
+  public void handleFullSyncNecessary() {
+    this.calledFullSyncNecessary = true;
   }
 
   @Override
   public void handleStageCompleted(Stage currentState,
            GlobalSession globalSession) {
     stageCounter--;
   }
 
--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
@@ -46,12 +46,17 @@ public class DefaultGlobalSessionCallbac
 
   @Override
   public void handleIncompleteStage(Stage currentState,
                                     GlobalSession globalSession) {
 
   }
 
   @Override
+  public void handleFullSyncNecessary() {
+
+  }
+
+  @Override
   public boolean shouldBackOffStorage() {
     return false;
   }
 }
--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/stage/test/TestFetchMetaGlobalStage.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/stage/test/TestFetchMetaGlobalStage.java
@@ -4,50 +4,59 @@
 package org.mozilla.gecko.sync.stage.test;
 
 import android.os.SystemClock;
 
 import org.json.simple.JSONArray;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mozilla.android.sync.net.test.TestGlobalSession;
 import org.mozilla.android.sync.net.test.TestMetaGlobal;
 import org.mozilla.android.sync.test.helpers.HTTPServerTestHelper;
 import org.mozilla.android.sync.test.helpers.MockGlobalSessionCallback;
 import org.mozilla.android.sync.test.helpers.MockServer;
 import org.mozilla.gecko.background.testhelpers.MockGlobalSession;
 import org.mozilla.gecko.background.testhelpers.TestRunner;
 import org.mozilla.gecko.background.testhelpers.WaitHelper;
 import org.mozilla.gecko.sync.AlreadySyncingException;
 import org.mozilla.gecko.sync.CollectionKeys;
 import org.mozilla.gecko.sync.CryptoRecord;
 import org.mozilla.gecko.sync.ExtendedJSONObject;
 import org.mozilla.gecko.sync.GlobalSession;
+import org.mozilla.gecko.sync.HTTPFailureException;
 import org.mozilla.gecko.sync.InfoCollections;
 import org.mozilla.gecko.sync.MetaGlobal;
 import org.mozilla.gecko.sync.NonObjectJSONException;
 import org.mozilla.gecko.sync.SyncConfigurationException;
 import org.mozilla.gecko.sync.crypto.CryptoException;
 import org.mozilla.gecko.sync.crypto.KeyBundle;
 import org.mozilla.gecko.sync.delegates.FreshStartDelegate;
+import org.mozilla.gecko.sync.delegates.GlobalSessionCallback;
 import org.mozilla.gecko.sync.delegates.KeyUploadDelegate;
 import org.mozilla.gecko.sync.delegates.WipeServerDelegate;
 import org.mozilla.gecko.sync.net.AuthHeaderProvider;
+import org.mozilla.gecko.sync.net.SyncStorageResponse;
 import org.mozilla.gecko.sync.stage.FetchMetaGlobalStage;
 import org.mozilla.gecko.sync.stage.GlobalSyncStage.Stage;
 import org.simpleframework.http.Request;
 import org.simpleframework.http.Response;
 
 import java.io.IOException;
 import java.net.URI;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import ch.boye.httpclientandroidlib.HttpResponse;
+import ch.boye.httpclientandroidlib.ProtocolVersion;
+import ch.boye.httpclientandroidlib.message.BasicHttpResponse;
+import ch.boye.httpclientandroidlib.message.BasicStatusLine;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 @RunWith(TestRunner.class)
 public class TestFetchMetaGlobalStage {
   @SuppressWarnings("unused")
   private static final String  LOG_TAG          = "TestMetaGlobalStage";
@@ -64,103 +73,115 @@ public class TestFetchMetaGlobalStage {
   private final String TEST_INFO_COLLECTIONS_JSON = "{}";
 
   private static final String  TEST_SYNC_ID         = "testSyncID";
   private static final long    TEST_STORAGE_VERSION = GlobalSession.STORAGE_VERSION;
 
   private InfoCollections infoCollections;
   private KeyBundle syncKeyBundle;
   private MockGlobalSessionCallback callback;
-  private GlobalSession session;
-
-  private boolean calledRequiresUpgrade = false;
-  private boolean calledProcessMissingMetaGlobal = false;
-  private boolean calledFreshStart = false;
-  private boolean calledWipeServer = false;
-  private boolean calledUploadKeys = false;
-  private boolean calledResetAllStages = false;
+  private LocalMockGlobalSession session;
 
   private static void assertSameContents(JSONArray expected, Set<String> actual) {
     assertEquals(expected.size(), actual.size());
     for (Object o : expected) {
       assertTrue(actual.contains(o));
     }
   }
 
+  private class LocalMockGlobalSession extends MockGlobalSession {
+    private boolean calledRequiresUpgrade = false;
+    private boolean calledProcessMissingMetaGlobal = false;
+    private boolean calledFreshStart = false;
+    private boolean calledWipeServer = false;
+    private boolean calledUploadKeys = false;
+    private boolean calledResetAllStages = false;
+    private boolean calledRestart = false;
+    private boolean calledAbort = false;
+
+    public LocalMockGlobalSession(String username, String password, KeyBundle keyBundle, GlobalSessionCallback callback) throws SyncConfigurationException, IllegalArgumentException, NonObjectJSONException, IOException {
+      super(username, password, keyBundle, callback);
+    }
+
+    @Override
+    protected void prepareStages() {
+      super.prepareStages();
+      withStage(Stage.fetchMetaGlobal, new FetchMetaGlobalStage());
+    }
+
+    @Override
+    public void requiresUpgrade() {
+      calledRequiresUpgrade = true;
+      this.abort(null, "Requires upgrade");
+    }
+
+    @Override
+    public void processMissingMetaGlobal(MetaGlobal mg) {
+      calledProcessMissingMetaGlobal = true;
+      this.abort(null, "Missing meta/global");
+    }
+
+    // Don't really uploadKeys.
+    @Override
+    public void uploadKeys(CollectionKeys keys, long lastModified, KeyUploadDelegate keyUploadDelegate) {
+      calledUploadKeys = true;
+      keyUploadDelegate.onKeysUploaded();
+    }
+
+    // On fresh start completed, just stop.
+    @Override
+    public void freshStart() {
+      calledFreshStart = true;
+      freshStart(this, new FreshStartDelegate() {
+        @Override
+        public void onFreshStartFailed(Exception e) {
+          WaitHelper.getTestWaiter().performNotify(e);
+        }
+
+        @Override
+        public void onFreshStart() {
+          WaitHelper.getTestWaiter().performNotify();
+        }
+      });
+    }
+
+    // Don't really wipeServer.
+    @Override
+    protected void wipeServer(final AuthHeaderProvider authHeaderProvider, final WipeServerDelegate wipeDelegate) {
+      calledWipeServer = true;
+      wipeDelegate.onWiped(System.currentTimeMillis());
+    }
+
+    @Override
+    protected void restart() throws AlreadySyncingException {
+      calledRestart = true;
+      WaitHelper.getTestWaiter().performNotify();
+    }
+
+    @Override
+    public void abort(Exception e, String reason) {
+      calledAbort = true;
+      super.abort(e, reason);
+    }
+
+    // Don't really resetAllStages.
+    @Override
+    public void resetAllStages() {
+      calledResetAllStages = true;
+    }
+  }
+
   @Before
   public void setUp() throws Exception {
-    calledRequiresUpgrade = false;
-    calledProcessMissingMetaGlobal = false;
-    calledFreshStart = false;
-    calledWipeServer = false;
-    calledUploadKeys = false;
-    calledResetAllStages = false;
-
     // Set info collections to not have crypto.
     infoCollections = new InfoCollections(new ExtendedJSONObject(TEST_INFO_COLLECTIONS_JSON));
 
     syncKeyBundle = new KeyBundle(TEST_USERNAME, TEST_SYNC_KEY);
     callback = new MockGlobalSessionCallback();
-    session = new MockGlobalSession(TEST_USERNAME, TEST_PASSWORD,
-      syncKeyBundle, callback) {
-      @Override
-      protected void prepareStages() {
-        super.prepareStages();
-        withStage(Stage.fetchMetaGlobal, new FetchMetaGlobalStage());
-      }
-
-      @Override
-      public void requiresUpgrade() {
-        calledRequiresUpgrade = true;
-        this.abort(null, "Requires upgrade");
-      }
-
-      @Override
-      public void processMissingMetaGlobal(MetaGlobal mg) {
-        calledProcessMissingMetaGlobal = true;
-        this.abort(null, "Missing meta/global");
-      }
-
-      // Don't really uploadKeys.
-      @Override
-      public void uploadKeys(CollectionKeys keys, KeyUploadDelegate keyUploadDelegate) {
-        calledUploadKeys = true;
-        keyUploadDelegate.onKeysUploaded();
-      }
-
-      // On fresh start completed, just stop.
-      @Override
-      public void freshStart() {
-        calledFreshStart = true;
-        freshStart(this, new FreshStartDelegate() {
-          @Override
-          public void onFreshStartFailed(Exception e) {
-            WaitHelper.getTestWaiter().performNotify(e);
-          }
-
-          @Override
-          public void onFreshStart() {
-            WaitHelper.getTestWaiter().performNotify();
-          }
-        });
-      }
-
-      // Don't really wipeServer.
-      @Override
-      protected void wipeServer(final AuthHeaderProvider authHeaderProvider, final WipeServerDelegate wipeDelegate) {
-        calledWipeServer = true;
-        wipeDelegate.onWiped(System.currentTimeMillis());
-      }
-
-      // Don't really resetAllStages.
-      @Override
-      public void resetAllStages() {
-        calledResetAllStages = true;
-      }
-    };
+    session = new LocalMockGlobalSession(TEST_USERNAME, TEST_PASSWORD, syncKeyBundle, callback);
     session.config.setClusterURL(new URI(TEST_CLUSTER_URL));
     session.config.infoCollections = infoCollections;
   }
 
   protected void doSession(MockServer server) {
     data.startHTTPServer(server);
     WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(new Runnable() {
       @Override
@@ -180,17 +201,17 @@ public class TestFetchMetaGlobalStage {
     MetaGlobal mg = new MetaGlobal(null, null);
     mg.setSyncID(TEST_SYNC_ID);
     mg.setStorageVersion(Long.valueOf(TEST_STORAGE_VERSION + 1));
 
     MockServer server = new MockServer(200, mg.asCryptoRecord().toJSONString());
     doSession(server);
 
     assertEquals(true, callback.calledError);
-    assertTrue(calledRequiresUpgrade);
+    assertTrue(session.calledRequiresUpgrade);
   }
 
   @SuppressWarnings("unchecked")
   private JSONArray makeTestDeclinedArray() {
     final JSONArray declined = new JSONArray();
     declined.add("foobar");
     return declined;
   }
@@ -212,18 +233,18 @@ public class TestFetchMetaGlobalStage {
     // Set declined engines in the server object.
     final JSONArray testingDeclinedEngines = makeTestDeclinedArray();
     mg.setDeclinedEngineNames(testingDeclinedEngines);
 
     MockServer server = new MockServer(200, mg.asCryptoRecord().toJSONString());
     doSession(server);
 
     assertTrue(callback.calledSuccess);
-    assertFalse(calledProcessMissingMetaGlobal);
-    assertFalse(calledResetAllStages);
+    assertFalse(session.calledProcessMissingMetaGlobal);
+    assertFalse(session.calledResetAllStages);
     assertEquals(TEST_SYNC_ID, session.config.metaGlobal.getSyncID());
     assertEquals(TEST_STORAGE_VERSION, session.config.metaGlobal.getStorageVersion().longValue());
     assertEquals(TEST_SYNC_ID, session.config.syncID);
 
     // Declined engines propagate from the server meta/global.
     final Set<String> actual = session.config.metaGlobal.getDeclinedEngineNames();
     assertSameContents(testingDeclinedEngines, actual);
   }
@@ -245,18 +266,18 @@ public class TestFetchMetaGlobalStage {
     // Set declined engines in the server object.
     final JSONArray testingDeclinedEngines = makeTestDeclinedArray();
     mg.setDeclinedEngineNames(testingDeclinedEngines);
 
     MockServer server = new MockServer(200, mg.asCryptoRecord().toJSONString());
     doSession(server);
 
     assertEquals(true, callback.calledSuccess);
-    assertFalse(calledProcessMissingMetaGlobal);
-    assertTrue(calledResetAllStages);
+    assertFalse(session.calledProcessMissingMetaGlobal);
+    assertTrue(session.calledResetAllStages);
     assertEquals(TEST_SYNC_ID, session.config.metaGlobal.getSyncID());
     assertEquals(TEST_STORAGE_VERSION, session.config.metaGlobal.getStorageVersion().longValue());
     assertEquals(TEST_SYNC_ID, session.config.syncID);
 
     // Declined engines propagate from the server meta/global.
     final Set<String> actual = session.config.metaGlobal.getDeclinedEngineNames();
     assertSameContents(testingDeclinedEngines, actual);
   }
@@ -294,41 +315,41 @@ public class TestFetchMetaGlobalStage {
   }
 
   @Test
   public void testFetchMissing() throws Exception {
     MockServer server = new MockServer(404, "missing");
     doSession(server);
 
     assertEquals(true, callback.calledError);
-    assertTrue(calledProcessMissingMetaGlobal);
+    assertTrue(session.calledProcessMissingMetaGlobal);
   }
 
   /**
    * Empty payload object has no syncID or storageVersion and should call freshStart.
    * @throws Exception
    */
   @Test
   public void testFetchEmptyPayload() throws Exception {
     MockServer server = new MockServer(200, TestMetaGlobal.TEST_META_GLOBAL_EMPTY_PAYLOAD_RESPONSE);
     doSession(server);
 
-    assertTrue(calledFreshStart);
+    assertTrue(session.calledFreshStart);
   }
 
   /**
    * No payload means no syncID or storageVersion and therefore we should call freshStart.
    * @throws Exception
    */
   @Test
   public void testFetchNoPayload() throws Exception {
     MockServer server = new MockServer(200, TestMetaGlobal.TEST_META_GLOBAL_NO_PAYLOAD_RESPONSE);
     doSession(server);
 
-    assertTrue(calledFreshStart);
+    assertTrue(session.calledFreshStart);
   }
 
   /**
    * Malformed payload is a server response issue, not a meta/global record
    * issue. This should error out of the sync.
    * @throws Exception
    */
   @Test
@@ -379,16 +400,67 @@ public class TestFetchMetaGlobalStage {
           // We shouldn't be trying to download anything after uploading meta/global.
           mgDownloaded.set(true);
         }
         this.handle(request, response, 404, "missing");
       }
     };
     doFreshStart(server);
 
-    assertTrue(this.calledFreshStart);
-    assertTrue(this.calledWipeServer);
-    assertTrue(this.calledUploadKeys);
+    assertTrue(session.calledFreshStart);
+    assertTrue(session.calledWipeServer);
+    assertTrue(session.calledUploadKeys);
     assertTrue(mgUploaded.get());
     assertFalse(mgDownloaded.get());
     assertEquals(GlobalSession.STORAGE_VERSION, uploadedMg.getStorageVersion().longValue());
   }
+
+  @Test
+  public void testFreshStartDelegateSuccess() {
+    final FreshStartDelegate freshStartDelegate = GlobalSession.makeFreshStartDelegate(session);
+
+    WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(
+            new Runnable() {
+              @Override
+              public void run() {
+                freshStartDelegate.onFreshStart();
+              }
+            }
+    ));
+
+    assertTrue(session.calledRestart);
+    assertFalse(session.calledAbort);
+  }
+
+  @Test
+  public void testFreshStartDelegate412() {
+    final FreshStartDelegate freshStartDelegate = GlobalSession.makeFreshStartDelegate(session);
+
+    WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(
+            new Runnable() {
+              @Override
+              public void run() {
+                freshStartDelegate.onFreshStartFailed(TestGlobalSession.makeHttpFailureException(412));
+              }
+            }
+    ));
+
+    assertTrue(session.calledRestart);
+    assertFalse(session.calledAbort);
+  }
+
+  @Test
+  public void testFreshStartDelegateNon412() {
+    final FreshStartDelegate freshStartDelegate = GlobalSession.makeFreshStartDelegate(session);
+
+    WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(
+            new Runnable() {
+              @Override
+              public void run() {
+                freshStartDelegate.onFreshStartFailed(TestGlobalSession.makeHttpFailureException(400));
+              }
+            }
+    ));
+
+    assertFalse(session.calledRestart);
+    assertTrue(session.calledAbort);
+  }
 }