Bug 684798 - Part 2: Sanitize wipeServer and catch all server maintenance errors. r=rnewman
authorPhilipp von Weitershausen <philipp@weitershausen.de>
Wed, 12 Oct 2011 13:53:13 -0700
changeset 79706 c7a5c385d01aa5fd4f38f4f74ecb9acb643bcbbc
parent 79705 f15a17ef38dd73dd123a9279b9e0533124ee2c72
child 79707 f06dfa65c0beca988e13f26644be2aa498446511
push id506
push userclegnitto@mozilla.com
push dateWed, 09 Nov 2011 02:03:18 +0000
treeherdermozilla-aurora@63587fc7bb93 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrnewman
bugs684798
milestone10.0a1
Bug 684798 - Part 2: Sanitize wipeServer and catch all server maintenance errors. r=rnewman
services/sync/modules/service.js
services/sync/tests/unit/head_http_server.js
services/sync/tests/unit/test_errorhandler.js
services/sync/tests/unit/test_service_sync_remoteSetup.js
services/sync/tests/unit/test_service_sync_updateEnabledEngines.js
services/sync/tests/unit/test_service_wipeServer.js
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -1555,72 +1555,99 @@ WeaveSvc.prototype = {
   },
 
   _freshStart: function WeaveSvc__freshStart() {
     this._log.info("Fresh start. Resetting client and considering key upgrade.");
     this.resetClient();
     CollectionKeys.clear();
     this.upgradeSyncKey(this.syncID);
 
+    // Wipe the server.
+    let wipeTimestamp = this.wipeServer();
+
+    // Upload a new meta/global record.
     let meta = new WBORecord("meta", "global");
     meta.payload.syncID = this.syncID;
     meta.payload.storageVersion = STORAGE_VERSION;
     meta.isNew = true;
 
     this._log.debug("New metadata record: " + JSON.stringify(meta.payload));
-    let resp = new Resource(this.metaURL).put(meta);
-    if (!resp.success)
+    let res = new Resource(this.metaURL);
+    // It would be good to set the X-If-Unmodified-Since header to `timestamp`
+    // for this PUT to ensure at least some level of transactionality.
+    // Unfortunately, the servers don't support it after a wipe right now
+    // (bug 693893), so we're going to defer this until bug 692700.
+    let resp = res.put(meta);
+    if (!resp.success) {
+      // If we got into a race condition, we'll abort the sync this way, too.
+      // That's fine. We'll just wait till the next sync. The client that we're
+      // racing is probably busy uploading stuff right now anyway.
       throw resp;
+    }
     Records.set(this.metaURL, meta);
 
     // Wipe everything we know about except meta because we just uploaded it
     let collections = [Clients].concat(Engines.getAll()).map(function(engine) {
       return engine.name;
     });
-    this.wipeServer(collections);
 
     // Generate, upload, and download new keys. Do this last so we don't wipe
     // them...
     this.generateNewSymmetricKeys();
   },
 
   /**
    * Wipe user data from the server.
    *
    * @param collections [optional]
-   *        Array of collections to wipe. If not given, all collections are wiped.
-   *
-   * @param includeKeys [optional]
-   *        If true, keys/pubkey and keys/privkey are deleted from the server.
-   *        This is false by default, which will cause the usual upgrade paths
-   *        to leave those keys on the server. This is to solve Bug 614737: old
-   *        clients check for keys *before* checking storage versions.
+   *        Array of collections to wipe. If not given, all collections are
+   *        wiped by issuing a DELETE request for `storageURL`.
    *
-   *        Note that this parameter only has an effect if `collections` is not
-   *        passed. If you explicitly pass a list of collections, they will be
-   *        processed regardless of the value of `includeKeys`.
+   * @return the server's timestamp of the (last) DELETE.
    */
-  wipeServer: function wipeServer(collections, includeKeyPairs)
+  wipeServer: function wipeServer(collections)
     this._notify("wipe-server", "", function() {
+      let response;
       if (!collections) {
-        collections = [];
-        let info = new Resource(this.infoURL).get();
-        for (let name in info.obj) {
-          if (includeKeyPairs || (name != "keys"))
-            collections.push(name);
+        // Strip the trailing slash.
+        let res = new Resource(this.storageURL.slice(0, -1));
+        res.setHeader("X-Confirm-Delete", "1");
+        try {
+          response = res.delete();
+        } catch (ex) {
+          this._log.debug("Failed to wipe server: " + Utils.exceptionStr(ex));
+          throw ex;
+        }
+        if (response.status != 200 && response.status != 404) {
+          this._log.debug("Aborting wipeServer. Server responded with " +
+                          response.status + " response for " + this.storageURL);
+          throw response;
+        }
+        return response.headers["x-weave-timestamp"];
+      }
+      let timestamp;
+      for each (let name in collections) {
+        let url = this.storageURL + name;
+        try {
+          response = new Resource(url).delete();
+        } catch (ex) {
+          this._log.debug("Failed to wipe '" + name + "' collection: " +
+                          Utils.exceptionStr(ex));
+          throw ex;
+        }
+        if (response.status != 200 && response.status != 404) {
+          this._log.debug("Aborting wipeServer. Server responded with " +
+                          response.status + " response for " + url);
+          throw response;
+        }
+        if ("x-weave-timestamp" in response.headers) {
+          timestamp = response.headers["x-weave-timestamp"];
         }
       }
-      for each (let name in collections) {
-        let url = this.storageURL + name;
-        let response = new Resource(url).delete();
-        if (response.status != 200 && response.status != 404) {
-          throw "Aborting wipeServer. Server responded with "
-                + response.status + " response for " + url;
-        }
-      }
+      return timestamp;
     })(),
 
   /**
    * Wipe all local user data.
    *
    * @param engines [optional]
    *        Array of engine names to wipe. If not given, all engines are used.
    */
--- a/services/sync/tests/unit/head_http_server.js
+++ b/services/sync/tests/unit/head_http_server.js
@@ -5,16 +5,27 @@ const SYNC_API_VERSION = "1.1";
 
 // Use the same method that record.js does, which mirrors the server.
 // The server returns timestamps with 1/100 sec granularity. Note that this is
 // subject to change: see Bug 650435.
 function new_timestamp() {
   return Math.round(Date.now() / 10) / 100;
 }
 
+function return_timestamp(request, response, timestamp) {
+  if (!timestamp) {
+    timestamp = new_timestamp();
+  }
+  let body = "" + timestamp;
+  response.setHeader("X-Weave-Timestamp", body);
+  response.setStatusLine(request.httpVersion, 200, "OK");
+  response.bodyOutputStream.write(body, body.length);
+  return timestamp;
+}
+
 function httpd_setup (handlers) {
   let server = new nsHttpServer();
   let port   = 8080;
   for (let path in handlers) {
     server.registerPathHandler(path, handlers[path]);
   }
   try {
     server.start(port);
@@ -603,17 +614,17 @@ SyncServer.prototype = {
   },
 
   /**
    * Return a server timestamp for a record.
    * The server returns timestamps with 1/100 sec granularity. Note that this is
    * subject to change: see Bug 650435.
    */
   timestamp: function timestamp() {
-    return Math.round(Date.now() / 10) / 100;
+    return new_timestamp();
   },
 
   /**
    * Create a new user, complete with an empty set of collections.
    *
    * @param username
    *        The username to use. An Error will be thrown if a user by that name
    *        already exists.
--- a/services/sync/tests/unit/test_errorhandler.js
+++ b/services/sync/tests/unit/test_errorhandler.js
@@ -73,33 +73,45 @@ function sync_httpd_setup() {
   let clientsColl = new ServerCollection({}, true);
 
   // Tracking info/collections.
   let collectionsHelper = track_collections_helper();
   let upd = collectionsHelper.with_updated_collection;
 
   let handler_401 = httpd_handler(401, "Unauthorized");
   return httpd_setup({
+    // Normal server behaviour.
     "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()),
     "/1.1/johndoe/info/collections": collectionsHelper.handler,
     "/1.1/johndoe/storage/crypto/keys":
       upd("crypto", (new ServerWBO("keys")).handler()),
     "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()),
 
+    // Credentials are wrong or node reallocated.
     "/1.1/janedoe/storage/meta/global": handler_401,
     "/1.1/janedoe/info/collections": handler_401,
 
-    "/maintenance/1.1/johnsmith/info/collections": service_unavailable,
+    // Maintenance or overloaded (503 + Retry-After) at info/collections.
+    "/maintenance/1.1/broken.info/info/collections": service_unavailable,
+
+    // Maintenance or overloaded (503 + Retry-After) at meta/global.
+    "/maintenance/1.1/broken.meta/storage/meta/global": service_unavailable,
+    "/maintenance/1.1/broken.meta/info/collections": collectionsHelper.handler,
 
-    "/maintenance/1.1/janesmith/storage/meta/global": service_unavailable,
-    "/maintenance/1.1/janesmith/info/collections": collectionsHelper.handler,
+    // Maintenance or overloaded (503 + Retry-After) at crypto/keys.
+    "/maintenance/1.1/broken.keys/storage/meta/global": upd("meta", global.handler()),
+    "/maintenance/1.1/broken.keys/info/collections": collectionsHelper.handler,
+    "/maintenance/1.1/broken.keys/storage/crypto/keys": service_unavailable,
 
-    "/maintenance/1.1/foo/storage/meta/global": upd("meta", global.handler()),
-    "/maintenance/1.1/foo/info/collections": collectionsHelper.handler,
-    "/maintenance/1.1/foo/storage/crypto/keys": service_unavailable,
+    // Maintenance or overloaded (503 + Retry-After) at wiping collection.
+    "/maintenance/1.1/broken.wipe/info/collections": collectionsHelper.handler,
+    "/maintenance/1.1/broken.wipe/storage/meta/global": upd("meta", global.handler()),
+    "/maintenance/1.1/broken.wipe/storage/crypto/keys":
+      upd("crypto", (new ServerWBO("keys")).handler()),
+    "/maintenance/1.1/broken.wipe/storage": service_unavailable
   });
 }
 
 function setUp() {
   Service.username = "johndoe";
   Service.password = "ilovejane";
   Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/";
@@ -727,19 +739,19 @@ add_test(function test_sync_server_maint
   Service.sync();
 });
 
 add_test(function test_info_collections_login_server_maintenance_error() {
   // Test info/collections server maintenance errors are not reported.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.info";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "johnsmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
 
   function onUIUpdate() {
     do_throw("Shouldn't get here!");
@@ -766,19 +778,19 @@ add_test(function test_info_collections_
   Service.sync();
 });
 
 add_test(function test_meta_global_login_server_maintenance_error() {
   // Test meta/global server maintenance errors are not reported.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.meta";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "janesmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
 
   function onUIUpdate() {
     do_throw("Shouldn't get here!");
@@ -805,18 +817,18 @@ add_test(function test_meta_global_login
   Service.sync();
 });
 
 add_test(function test_crypto_keys_login_server_maintenance_error() {
   // Test crypto/keys server maintenance errors are not reported.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.keys";
   Service.clusterURL = "http://localhost:8080/maintenance/";
-  Service.username = "foo";
   // Force re-download of keys
   CollectionKeys.clear();
 
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
@@ -872,19 +884,19 @@ add_test(function test_sync_prolonged_se
   Service.sync();
 });
 
 add_test(function test_info_collections_login_prolonged_server_maintenance_error(){
   // Test info/collections prolonged server maintenance errors are reported.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.info";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "johnsmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
 
   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
@@ -904,19 +916,19 @@ add_test(function test_info_collections_
   Service.sync();
 });
 
 add_test(function test_meta_global_login_prolonged_server_maintenance_error(){
   // Test meta/global prolonged server maintenance errors are reported.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.meta";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "janesmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
 
   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
@@ -936,18 +948,18 @@ add_test(function test_meta_global_login
   Service.sync();
 });
 
 add_test(function test_download_crypto_keys_login_prolonged_server_maintenance_error(){
   // Test crypto/keys prolonged server maintenance errors are reported.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.keys";
   Service.clusterURL = "http://localhost:8080/maintenance/";
-  Service.username = "foo";
   // Force re-download of keys
   CollectionKeys.clear();
 
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
@@ -970,17 +982,51 @@ add_test(function test_download_crypto_k
   Service.sync();
 });
 
 add_test(function test_upload_crypto_keys_login_prolonged_server_maintenance_error(){
   // Test crypto/keys prolonged server maintenance errors are reported.
   let server = sync_httpd_setup();
 
   // Start off with an empty account, do not upload a key.
-  Service.username = "foo";
+  Service.username = "broken.keys";
+  Service.password = "ilovejane";
+  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
+  Service.clusterURL = "http://localhost:8080/maintenance/";
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+    Svc.Obs.remove("weave:service:backoff:interval", observe);
+    backoffInterval = subject;
+  });
+
+  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+    do_check_true(Status.enforceBackoff);
+    do_check_eq(backoffInterval, 42);
+    do_check_eq(Status.service, SYNC_FAILED);
+    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
+
+    clean();
+    server.stop(run_next_test);
+  });
+
+  do_check_false(Status.enforceBackoff);
+  do_check_eq(Status.service, STATUS_OK);
+
+  setLastSync(PROLONGED_ERROR_DURATION);
+  Service.sync();
+});
+
+add_test(function test_wipeServer_login_prolonged_server_maintenance_error(){
+  // Test crypto/keys prolonged server maintenance errors are reported.
+  let server = sync_httpd_setup();
+
+  // Start off with an empty account, do not upload a key.
+  Service.username = "broken.wipe";
   Service.password = "ilovejane";
   Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
@@ -1032,19 +1078,19 @@ add_test(function test_sync_syncAndRepor
 });
 
 add_test(function test_info_collections_login_syncAndReportErrors_server_maintenance_error() {
   // Test info/collections server maintenance errors are reported
   // when calling syncAndReportErrors.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.info";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "johnsmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
 
   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
@@ -1065,19 +1111,19 @@ add_test(function test_info_collections_
 });
 
 add_test(function test_meta_global_login_syncAndReportErrors_server_maintenance_error() {
   // Test meta/global server maintenance errors are reported
   // when calling syncAndReportErrors.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.meta";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "janesmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
 
   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
@@ -1098,18 +1144,18 @@ add_test(function test_meta_global_login
 });
 
 add_test(function test_download_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
   // Test crypto/keys server maintenance errors are reported
   // when calling syncAndReportErrors.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.keys";
   Service.clusterURL = "http://localhost:8080/maintenance/";
-  Service.username = "foo";
   // Force re-download of keys
   CollectionKeys.clear();
 
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
@@ -1133,17 +1179,52 @@ add_test(function test_download_crypto_k
 });
 
 add_test(function test_upload_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
   // Test crypto/keys server maintenance errors are reported
   // when calling syncAndReportErrors.
   let server = sync_httpd_setup();
 
   // Start off with an empty account, do not upload a key.
-  Service.username = "foo";
+  Service.username = "broken.keys";
+  Service.password = "ilovejane";
+  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
+  Service.clusterURL = "http://localhost:8080/maintenance/";
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+    Svc.Obs.remove("weave:service:backoff:interval", observe);
+    backoffInterval = subject;
+  });
+
+  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+    do_check_true(Status.enforceBackoff);
+    do_check_eq(backoffInterval, 42);
+    do_check_eq(Status.service, LOGIN_FAILED);
+    do_check_eq(Status.login, SERVER_MAINTENANCE);
+
+    clean();
+    server.stop(run_next_test);
+  });
+
+  do_check_false(Status.enforceBackoff);
+  do_check_eq(Status.service, STATUS_OK);
+
+  setLastSync(NON_PROLONGED_ERROR_DURATION);
+  ErrorHandler.syncAndReportErrors();
+});
+
+add_test(function test_wipeServer_login_syncAndReportErrors_server_maintenance_error() {
+  // Test crypto/keys server maintenance errors are reported
+  // when calling syncAndReportErrors.
+  let server = sync_httpd_setup();
+
+  // Start off with an empty account, do not upload a key.
+  Service.username = "broken.wipe";
   Service.password = "ilovejane";
   Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
@@ -1195,19 +1276,19 @@ add_test(function test_sync_syncAndRepor
 });
 
 add_test(function test_info_collections_login_syncAndReportErrors_prolonged_server_maintenance_error() {
   // Test info/collections server maintenance errors are reported
   // when calling syncAndReportErrors.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.info";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "johnsmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
 
   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
@@ -1228,19 +1309,19 @@ add_test(function test_info_collections_
 });
 
 add_test(function test_meta_global_login_syncAndReportErrors_prolonged_server_maintenance_error() {
   // Test meta/global server maintenance errors are reported
   // when calling syncAndReportErrors.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.meta";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
-  Service.username = "janesmith";
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
 
   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
@@ -1261,18 +1342,18 @@ add_test(function test_meta_global_login
 });
 
 add_test(function test_download_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
   // Test crypto/keys server maintenance errors are reported
   // when calling syncAndReportErrors.
   let server = sync_httpd_setup();
   setUp();
 
+  Service.username = "broken.keys";
   Service.clusterURL = "http://localhost:8080/maintenance/";
-  Service.username = "foo";
   // Force re-download of keys
   CollectionKeys.clear();
 
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
   });
@@ -1296,17 +1377,52 @@ add_test(function test_download_crypto_k
 });
 
 add_test(function test_upload_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
   // Test crypto/keys server maintenance errors are reported
   // when calling syncAndReportErrors.
   let server = sync_httpd_setup();
 
   // Start off with an empty account, do not upload a key.
-  Service.username = "foo";
+  Service.username = "broken.keys";
+  Service.password = "ilovejane";
+  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
+  Service.clusterURL = "http://localhost:8080/maintenance/";
+
+  let backoffInterval;
+  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+    Svc.Obs.remove("weave:service:backoff:interval", observe);
+    backoffInterval = subject;
+  });
+
+  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+    do_check_true(Status.enforceBackoff);
+    do_check_eq(backoffInterval, 42);
+    do_check_eq(Status.service, LOGIN_FAILED);
+    do_check_eq(Status.login, SERVER_MAINTENANCE);
+
+    clean();
+    server.stop(run_next_test);
+  });
+
+  do_check_false(Status.enforceBackoff);
+  do_check_eq(Status.service, STATUS_OK);
+
+  setLastSync(PROLONGED_ERROR_DURATION);
+  ErrorHandler.syncAndReportErrors();
+});
+
+add_test(function test_wipeServer_login_syncAndReportErrors_prolonged_server_maintenance_error() {
+  // Test crypto/keys server maintenance errors are reported
+  // when calling syncAndReportErrors.
+  let server = sync_httpd_setup();
+
+  // Start off with an empty account, do not upload a key.
+  Service.username = "broken.wipe";
   Service.password = "ilovejane";
   Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/maintenance/";
 
   let backoffInterval;
   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
     Svc.Obs.remove("weave:service:backoff:interval", observe);
     backoffInterval = subject;
--- a/services/sync/tests/unit/test_service_sync_remoteSetup.js
+++ b/services/sync/tests/unit/test_service_sync_remoteSetup.js
@@ -24,17 +24,38 @@ function run_test() {
       handler.apply(this, arguments);
     };
   }
 
   let keysWBO = new ServerWBO("keys");
   let cryptoColl = new ServerCollection({keys: keysWBO});
   let metaColl = new ServerCollection({global: meta_global});
   do_test_pending();
+
+  /**
+   * Handle the bulk DELETE request sent by wipeServer.
+   */
+  function storageHandler(request, response) {
+    do_check_eq("DELETE", request.method);
+    do_check_true(request.hasHeader("X-Confirm-Delete"));
+
+    _("Wiping out all collections.");
+    cryptoColl.delete({});
+    clients.delete({});
+    metaColl.delete({});
+
+    let ts = new_timestamp();
+    collectionsHelper.update_collection("crypto", ts);
+    collectionsHelper.update_collection("clients", ts);
+    collectionsHelper.update_collection("meta", ts);
+    return_timestamp(request, response, ts);
+  }
+
   let server = httpd_setup({
+    "/1.1/johndoe/storage": storageHandler,
     "/1.1/johndoe/storage/crypto/keys": upd("crypto", keysWBO.handler()),
     "/1.1/johndoe/storage/crypto": upd("crypto", cryptoColl.handler()),
     "/1.1/johndoe/storage/clients": upd("clients", clients.handler()),
     "/1.1/johndoe/storage/meta/global": upd("meta", wasCalledHandler(meta_global)),
     "/1.1/johndoe/storage/meta": upd("meta", wasCalledHandler(metaColl)),
     "/1.1/johndoe/info/collections": collectionsHelper.handler
   });
 
--- a/services/sync/tests/unit/test_service_sync_updateEnabledEngines.js
+++ b/services/sync/tests/unit/test_service_sync_updateEnabledEngines.js
@@ -61,40 +61,53 @@ function sync_httpd_setup(handlers) {
     upd("clients", cl.handler());
   
   return httpd_setup(handlers);
 }
 
 function setUp() {
   Service.username = "johndoe";
   Service.password = "ilovejane";
-  Service.passphrase = "sekrit";
+  Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/";
+  // So that we can poke at meta/global.
   new FakeCryptoService();
+
+  // Ensure that the server has valid keys so that logging in will work and not
+  // result in a server wipe, rendering many of these tests useless.
+  generateNewKeys();
+  let serverKeys = CollectionKeys.asWBO("crypto", "keys");
+  serverKeys.encrypt(Service.syncKeyBundle);
+  return serverKeys.upload(Service.cryptoKeysURL).success;
 }
 
 const PAYLOAD = 42;
 
+
+function run_test() {
+  initTestLogging("Trace");
+  run_next_test();
+}
+
 add_test(function test_newAccount() {
   _("Test: New account does not disable locally enabled engines.");
   let engine = Engines.get("steam");
   let server = sync_httpd_setup({
     "/1.1/johndoe/storage/meta/global": new ServerWBO("global", {}).handler(),
     "/1.1/johndoe/storage/steam": new ServerWBO("steam", {}).handler()
   });
   setUp();
 
   try {
     _("Engine is enabled from the beginning.");
     Service._ignorePrefObserver = true;
     engine.enabled = true;
     Service._ignorePrefObserver = false;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Engine continues to be enabled.");
     do_check_true(engine.enabled);
   } finally {
     Service.startOver();
     server.stop(run_next_test);
   }
@@ -113,17 +126,16 @@ add_test(function test_enabledLocally() 
   });
   setUp();
 
   try {
     _("Enable engine locally.");
     engine.enabled = true;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Meta record now contains the new engine.");
     do_check_true(!!metaWBO.data.engines.steam);
 
     _("Engine continues to be enabled.");
     do_check_true(engine.enabled);
   } finally {
@@ -138,31 +150,31 @@ add_test(function test_disabledLocally()
   let engine = Engines.get("steam");
   let metaWBO = new ServerWBO("global", {
     syncID: Service.syncID,
     storageVersion: STORAGE_VERSION,
     engines: {steam: {syncID: engine.syncID,
                       version: engine.version}}
   });
   let steamCollection = new ServerWBO("steam", PAYLOAD);
+
   let server = sync_httpd_setup({
     "/1.1/johndoe/storage/meta/global": metaWBO.handler(),
     "/1.1/johndoe/storage/steam": steamCollection.handler()
   });
   setUp();
 
   try {
     _("Disable engine locally.");
     Service._ignorePrefObserver = true;
     engine.enabled = true;
     Service._ignorePrefObserver = false;
     engine.enabled = false;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Meta record no longer contains engine.");
     do_check_false(!!metaWBO.data.engines.steam);
 
     _("Server records are wiped.");
     do_check_eq(steamCollection.payload, undefined);
 
@@ -200,17 +212,16 @@ add_test(function test_enabledRemotely()
     let wbo = CollectionKeys.generateNewKeysWBO();
     wbo.encrypt(Service.syncKeyBundle);
     do_check_eq(200, wbo.upload(Service.cryptoKeysURL).status);
 
     _("Engine is disabled.");
     do_check_false(engine.enabled);
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Engine is enabled.");
     do_check_true(engine.enabled);
 
     _("Meta record still present.");
     do_check_eq(metaWBO.data.engines.steam.syncID, engine.syncID);
   } finally {
@@ -237,17 +248,16 @@ add_test(function test_disabledRemotelyT
 
   try {
     _("Enable engine locally.");
     Service._ignorePrefObserver = true;
     engine.enabled = true;
     Service._ignorePrefObserver = false;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Disable engine by deleting from meta/global.");
     let d = metaWBO.data; 
     delete d.engines["steam"];
     metaWBO.payload = JSON.stringify(d);
     metaWBO.modified = Date.now() / 1000;
     
@@ -279,17 +289,16 @@ add_test(function test_disabledRemotely(
 
   try {
     _("Enable engine locally.");
     Service._ignorePrefObserver = true;
     engine.enabled = true;
     Service._ignorePrefObserver = false;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Engine is not disabled: only one client.");
     do_check_true(engine.enabled);
     
   } finally {
     Service.startOver();
     server.stop(run_next_test);
@@ -311,17 +320,16 @@ add_test(function test_dependentEnginesE
   });
   setUp();
 
   try {
     _("Enable engine locally. Doing it on one is enough.");
     steamEngine.enabled = true;
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Meta record now contains the new engines.");
     do_check_true(!!metaWBO.data.engines.steam);
     do_check_true(!!metaWBO.data.engines.stirling);
 
     _("Engines continue to be enabled.");
     do_check_true(steamEngine.enabled);
@@ -343,16 +351,17 @@ add_test(function test_dependentEnginesD
     engines: {steam: {syncID: steamEngine.syncID,
                       version: steamEngine.version},
               stirling: {syncID: stirlingEngine.syncID,
                          version: stirlingEngine.version}}
   });
 
   let steamCollection = new ServerWBO("steam", PAYLOAD);
   let stirlingCollection = new ServerWBO("stirling", PAYLOAD);
+
   let server = sync_httpd_setup({
     "/1.1/johndoe/storage/meta/global":     metaWBO.handler(),
     "/1.1/johndoe/storage/steam":           steamCollection.handler(),
     "/1.1/johndoe/storage/stirling":        stirlingCollection.handler()
   });
   setUp();
 
   try {
@@ -360,17 +369,16 @@ add_test(function test_dependentEnginesD
     Service._ignorePrefObserver = true;
     steamEngine.enabled = true;
     do_check_true(stirlingEngine.enabled);
     Service._ignorePrefObserver = false;
     steamEngine.enabled = false;
     do_check_false(stirlingEngine.enabled);
 
     _("Sync.");
-    Weave.Service.login();
     Weave.Service.sync();
 
     _("Meta record no longer contains engines.");
     do_check_false(!!metaWBO.data.engines.steam);
     do_check_false(!!metaWBO.data.engines.stirling);
 
     _("Server records are wiped.");
     do_check_eq(steamCollection.payload, undefined);
@@ -379,12 +387,8 @@ add_test(function test_dependentEnginesD
     _("Engines continue to be disabled.");
     do_check_false(steamEngine.enabled);
     do_check_false(stirlingEngine.enabled);
   } finally {
     Service.startOver();
     server.stop(run_next_test);
   }
 });
-
-function run_test() {
-  run_next_test();
-}
--- a/services/sync/tests/unit/test_service_wipeServer.js
+++ b/services/sync/tests/unit/test_service_wipeServer.js
@@ -8,137 +8,214 @@ Cu.import("resource://services-sync/serv
 function FakeCollection() {
   this.deleted = false;
 }
 FakeCollection.prototype = {
   handler: function() {
     let self = this;
     return function(request, response) {
       let body = "";
+      self.timestamp = new_timestamp();
+      let timestamp = "" + self.timestamp;
       if (request.method == "DELETE") {
-          body = JSON.stringify(Date.now() / 1000);
+          body = timestamp;
           self.deleted = true;
       }
+      response.setHeader("X-Weave-Timestamp", timestamp);
       response.setStatusLine(request.httpVersion, 200, "OK");
       response.bodyOutputStream.write(body, body.length);
     };
   }
 };
 
-function serviceUnavailable(request, response) {
-  let body = "Service Unavailable";
-  response.setStatusLine(request.httpVersion, 503, "Service Unavailable");
-  response.bodyOutputStream.write(body, body.length);
-}
-
 function setUpTestFixtures() {
   let cryptoService = new FakeCryptoService();
 
   Service.clusterURL = "http://localhost:8080/";
   Service.username = "johndoe";
   Service.passphrase = "aabcdeabcdeabcdeabcdeabcde";
 }
 
-function test_withCollectionList_fail() {
+
+function run_test() {
+  initTestLogging("Trace");
+  run_next_test();
+}
+
+add_test(function test_wipeServer_list_success() {
   _("Service.wipeServer() deletes collections given as argument.");
 
   let steam_coll = new FakeCollection();
   let diesel_coll = new FakeCollection();
 
   let server = httpd_setup({
     "/1.1/johndoe/storage/steam": steam_coll.handler(),
-    "/1.1/johndoe/storage/petrol": serviceUnavailable,
+    "/1.1/johndoe/storage/diesel": diesel_coll.handler(),
+    "/1.1/johndoe/storage/petrol": httpd_handler(404, "Not Found")
+  });
+
+  try {
+    setUpTestFixtures();
+
+    _("Confirm initial environment.");
+    do_check_false(steam_coll.deleted);
+    do_check_false(diesel_coll.deleted);
+
+    _("wipeServer() will happily ignore the non-existent collection and use the timestamp of the last DELETE that was successful.");
+    let timestamp = Service.wipeServer(["steam", "diesel", "petrol"]);
+    do_check_eq(timestamp, diesel_coll.timestamp);
+
+    _("wipeServer stopped deleting after encountering an error with the 'petrol' collection, thus only 'steam' has been deleted.");
+    do_check_true(steam_coll.deleted);
+    do_check_true(diesel_coll.deleted);
+
+  } finally {
+    server.stop(run_next_test);
+    Svc.Prefs.resetBranch("");
+  }
+});
+
+add_test(function test_wipeServer_list_503() {
+  _("Service.wipeServer() deletes collections given as argument.");
+
+  let steam_coll = new FakeCollection();
+  let diesel_coll = new FakeCollection();
+
+  let server = httpd_setup({
+    "/1.1/johndoe/storage/steam": steam_coll.handler(),
+    "/1.1/johndoe/storage/petrol": httpd_handler(503, "Service Unavailable"),
     "/1.1/johndoe/storage/diesel": diesel_coll.handler()
   });
-  do_test_pending();
 
   try {
     setUpTestFixtures();
 
     _("Confirm initial environment.");
     do_check_false(steam_coll.deleted);
     do_check_false(diesel_coll.deleted);
 
     _("wipeServer() will happily ignore the non-existent collection, delete the 'steam' collection and abort after an receiving an error on the 'petrol' collection.");
     let error;
     try {
       Service.wipeServer(["non-existent", "steam", "petrol", "diesel"]);
+      do_throw("Should have thrown!");
     } catch(ex) {
       error = ex;
     }
     _("wipeServer() threw this exception: " + error);
-    do_check_true(error != undefined);
+    do_check_eq(error.status, 503);
 
     _("wipeServer stopped deleting after encountering an error with the 'petrol' collection, thus only 'steam' has been deleted.");
     do_check_true(steam_coll.deleted);
     do_check_false(diesel_coll.deleted);
 
   } finally {
-    server.stop(do_test_finished);
+    server.stop(run_next_test);
     Svc.Prefs.resetBranch("");
   }
-}
+});
 
-function test_wipeServer_leaves_collections() {
-  _("Service.wipeServer() deletes everything but keys.");
+add_test(function test_wipeServer_all_success() {
+  _("Service.wipeServer() deletes all the things.");
   
-  let steam_coll = new FakeCollection();
-  let diesel_coll = new FakeCollection();
-  let keys_coll = new FakeCollection();
+  /**
+   * Handle the bulk DELETE request sent by wipeServer.
+   */
+  let deleted = false;
+  let serverTimestamp;
+  function storageHandler(request, response) {
+    do_check_eq("DELETE", request.method);
+    do_check_true(request.hasHeader("X-Confirm-Delete"));
+    deleted = true;
+    serverTimestamp = return_timestamp(request, response);
+  }
+
+  let server = httpd_setup({
+    "/1.1/johndoe/storage": storageHandler
+  });
+  setUpTestFixtures();
 
-  function info_collections(request, response) {
-    let collections = {};
-    let timestamp = Date.now() / 1000;
-    if (!steam_coll.deleted)
-      collections.steam = timestamp
-    if (!diesel_coll.deleted)
-      collections.diesel = timestamp;
-    if (!keys_coll.deleted)
-      collections.keys = timestamp;
-    let body = JSON.stringify(collections);
-    response.setStatusLine(request.httpVersion, 200, "OK");
-    response.bodyOutputStream.write(body, body.length);
+  _("Try deletion.");
+  let returnedTimestamp = Service.wipeServer();
+  do_check_true(deleted);
+  do_check_eq(returnedTimestamp, serverTimestamp);
+
+  server.stop(run_next_test);
+  Svc.Prefs.resetBranch("");
+});
+
+add_test(function test_wipeServer_all_404() {
+  _("Service.wipeServer() accepts a 404.");
+  
+  /**
+   * Handle the bulk DELETE request sent by wipeServer. Returns a 404.
+   */
+  let deleted = false;
+  let serverTimestamp;
+  function storageHandler(request, response) {
+    do_check_eq("DELETE", request.method);
+    do_check_true(request.hasHeader("X-Confirm-Delete"));
+    deleted = true;
+    serverTimestamp = new_timestamp();
+    response.setHeader("X-Weave-Timestamp", "" + serverTimestamp);
+    response.setStatusLine(request.httpVersion, 404, "Not Found");
   }
 
   let server = httpd_setup({
-    "/1.1/johndoe/storage/steam": steam_coll.handler(),
-    "/1.1/johndoe/storage/diesel": diesel_coll.handler(),
-    "/1.1/johndoe/storage/keys": keys_coll.handler(),
-    "/1.1/johndoe/info/collections": info_collections
+    "/1.1/johndoe/storage": storageHandler
   });
-  do_test_pending();
+  setUpTestFixtures();
+
+  _("Try deletion.");
+  let returnedTimestamp = Service.wipeServer();
+  do_check_true(deleted);
+  do_check_eq(returnedTimestamp, serverTimestamp);
+
+  server.stop(run_next_test);
+  Svc.Prefs.resetBranch("");
+});
 
+add_test(function test_wipeServer_all_503() {
+  _("Service.wipeServer() throws if it encounters a non-200/404 response.");
+  
+  /**
+   * Handle the bulk DELETE request sent by wipeServer. Returns a 503.
+   */
+  function storageHandler(request, response) {
+    do_check_eq("DELETE", request.method);
+    do_check_true(request.hasHeader("X-Confirm-Delete"));
+    response.setStatusLine(request.httpVersion, 503, "Service Unavailable");
+  }
+
+  let server = httpd_setup({
+    "/1.1/johndoe/storage": storageHandler
+  });
+  setUpTestFixtures();
+
+  _("Try deletion.");
+  let error;
   try {
-    setUpTestFixtures();
-    _("Info URL: " + Service.infoURL);
-
-    _("Confirm initial environment.");
-    do_check_false(steam_coll.deleted);
-    do_check_false(diesel_coll.deleted);
-    do_check_false(keys_coll.deleted);
-    
-    _("Collections: " + new Resource(Service.infoURL).get());
-    _("Try deletion.");
     Service.wipeServer();
-    _("Collections: " + new Resource(Service.infoURL).get());
-    
-    _("Make sure keys is still present.");
-    do_check_true(steam_coll.deleted);
-    do_check_true(diesel_coll.deleted);
-    do_check_false(keys_coll.deleted);
-    
-    _("Delete everything.");
-    Service.wipeServer(null, true);
-    do_check_true(steam_coll.deleted);
-    do_check_true(diesel_coll.deleted);
-    do_check_true(keys_coll.deleted);
-    
-  } finally {
-    server.stop(do_test_finished);
-    Svc.Prefs.resetBranch("");
+    do_throw("Should have thrown!");
+  } catch (ex) {
+    error = ex;
   }
-}
+  do_check_eq(error.status, 503);
+
+  server.stop(run_next_test);
+  Svc.Prefs.resetBranch("");
+});
+
+add_test(function test_wipeServer_all_connectionRefused() {
+  _("Service.wipeServer() throws if it encounters a network problem.");
+  setUpTestFixtures();
 
-function run_test() {
-  initTestLogging("Trace");
-  test_withCollectionList_fail();
-  test_wipeServer_leaves_collections();
-}
+  _("Try deletion.");
+  try {
+    Service.wipeServer();
+    do_throw("Should have thrown!");
+  } catch (ex) {
+    do_check_eq(ex.result, Cr.NS_ERROR_CONNECTION_REFUSED);
+  }
+
+  run_next_test();
+  Svc.Prefs.resetBranch("");
+});