Bug 659067 - Part 2: Move _syncError to ErrorHandler. r=philikon
authorMarina Samuel <msamuel@mozilla.com>
Fri, 26 Aug 2011 14:01:35 -0700
changeset 76305 bdd0d53dcaff2e0616b08086aa575de4697fe24c
parent 76304 298e7535687c453f49f4185ded4011afa3fdfb3f
child 76306 4778b8babec66927817f5161ac8545c5b65bc3f4
push id3
push userfelipc@gmail.com
push dateFri, 30 Sep 2011 20:09:13 +0000
reviewersphilikon
bugs659067
milestone9.0a1
Bug 659067 - Part 2: Move _syncError to ErrorHandler. r=philikon
browser/base/content/browser-syncui.js
services/sync/modules/policies.js
services/sync/modules/service.js
services/sync/tests/unit/test_errorhandler.js
services/sync/tests/unit/test_errorhandler_sync_checkServerError.js
services/sync/tests/unit/test_syncscheduler.js
--- a/browser/base/content/browser-syncui.js
+++ b/browser/base/content/browser-syncui.js
@@ -35,17 +35,18 @@
 # and other provisions required by the GPL or the LGPL. If you do not delete
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK *****
 
 // gSyncUI handles updating the tools menu
 let gSyncUI = {
-  _obs: ["weave:service:sync:start",
+  _obs: ["weave:engine:sync:error",
+         "weave:service:sync:start",
          "weave:service:sync:finish",
          "weave:service:sync:error",
          "weave:service:sync:delayed",
          "weave:service:quota:remaining",
          "weave:service:setup-complete",
          "weave:service:login:start",
          "weave:service:login:finish",
          "weave:service:login:error",
@@ -432,16 +433,17 @@ let gSyncUI = {
 
     switch (topic) {
       case "weave:service:sync:start":
         this.onActivityStart();
         break;
       case "weave:service:sync:finish":
         this.onSyncFinish();
         break;
+      case "weave:engine:sync:error":
       case "weave:service:sync:error":
         this.onSyncError();
         break;
       case "weave:service:sync:delayed":
         this.onSyncDelay();
         break;
       case "weave:service:quota:remaining":
         this.onQuotaNotice();
--- a/services/sync/modules/policies.js
+++ b/services/sync/modules/policies.js
@@ -83,16 +83,17 @@ let SyncScheduler = {
   init: function init() {
     this._log.level = Log4Moz.Level[Svc.Prefs.get("log.logger.service.main")];
     this.setDefaults();
     Svc.Obs.add("weave:engine:score:updated", this);
     Svc.Obs.add("network:offline-status-changed", this);
     Svc.Obs.add("weave:service:sync:start", this);
     Svc.Obs.add("weave:service:sync:finish", this);
     Svc.Obs.add("weave:engine:sync:finish", this);
+    Svc.Obs.add("weave:engine:sync:error", this);
     Svc.Obs.add("weave:service:login:error", this);
     Svc.Obs.add("weave:service:logout:finish", this);
     Svc.Obs.add("weave:service:sync:error", this);
     Svc.Obs.add("weave:service:backoff:interval", this);
     Svc.Obs.add("weave:service:ready", this);
     Svc.Obs.add("weave:engine:sync:applied", this);
     Svc.Obs.add("weave:service:setup-complete", this);
     Svc.Obs.add("weave:service:start-over", this);
@@ -124,30 +125,43 @@ let SyncScheduler = {
         Status.resetBackoff();
 
         this.globalScore = 0;
         break;
       case "weave:service:sync:finish":
         this.adjustSyncInterval();
 
         let sync_interval;
+
+        if (Status.service == SYNC_FAILED_PARTIAL && this.requiresBackoff) {
+          this.requiresBackoff = false;
+          this.handleSyncError();
+          return;
+        }
+
         this._syncErrors = 0;
-
         if (Status.sync == NO_SYNC_NODE_FOUND) {
           this._log.trace("Scheduling a sync at interval NO_SYNC_NODE_FOUND.");
           sync_interval = NO_SYNC_NODE_INTERVAL;
         }
         this.scheduleNextSync(sync_interval);
         break;
       case "weave:engine:sync:finish":
         if (data == "clients") {
           // Update the client mode because it might change what we sync.
           this.updateClientMode();
         }
         break;
+      case "weave:engine:sync:error":
+        // subject is the exception thrown by an engine's sync() method
+        let exception = subject;
+        if (exception.status >= 500 && exception.status <= 504) {
+          this.requiresBackoff = true;
+        }
+        break;
       case "weave:service:login:error":
         this.clearSyncTriggers();
         
         // Try again later, just as if we threw an error... only without the
         // error count.
         if (Status.login == MASTER_PASSWORD_LOCKED) {
           this._log.debug("Couldn't log in: master password is locked.");
           this._log.trace("Scheduling a sync at MASTER_PASSWORD_LOCKED_RETRY_INTERVAL");
@@ -415,16 +429,18 @@ let SyncScheduler = {
 };
 
 const LOG_PREFIX_SUCCESS = "success-";
 const LOG_PREFIX_ERROR   = "error-";
 
 let ErrorHandler = {
 
   init: function init() {
+    Svc.Obs.add("weave:engine:sync:applied", this);
+    Svc.Obs.add("weave:engine:sync:error", this);
     Svc.Obs.add("weave:service:login:error", this);
     Svc.Obs.add("weave:service:sync:error", this);
     Svc.Obs.add("weave:service:sync:finish", this);
 
     this.initLogs();
   },
 
   initLogs: function initLogs() {
@@ -445,16 +461,34 @@ let ErrorHandler = {
 
     let fapp = this._logAppender = new Log4Moz.StorageStreamAppender(formatter);
     fapp.level = Log4Moz.Level[Svc.Prefs.get("log.appender.file.level")];
     root.addAppender(fapp);
   },
 
   observe: function observe(subject, topic, data) {
     switch(topic) {
+      case "weave:engine:sync:applied":
+        if (subject.newFailed) {
+          // An engine isn't able to apply one or more incoming records.
+          // We don't fail hard on this, but it usually indicates a bug,
+          // so for now treat it as sync error (c.f. Service._syncEngine())
+          Status.engines = [data, ENGINE_APPLY_FAIL];
+          this._log.debug(data + " failed to apply some records.");
+        }
+        break;
+      case "weave:engine:sync:error":
+        let exception = subject;  // exception thrown by engine's sync() method
+        let engine_name = data;   // engine name that threw the exception
+
+        this.checkServerError(exception);
+
+        Status.engines = [engine_name, exception.failureCode || ENGINE_UNKNOWN_FAIL];
+        this._log.debug(engine_name + " failed: " + Utils.exceptionStr(exception));
+        break;
       case "weave:service:login:error":
         if (Status.login == LOGIN_FAILED_NETWORK_ERROR &&
             !Services.io.offline) {
           this._ignorableErrorCount += 1;
         } else {
           this.resetFileLog(Svc.Prefs.get("log.appender.file.logOnError"),
                             LOG_PREFIX_ERROR);
         }
@@ -471,18 +505,24 @@ let ErrorHandler = {
             break;
           default:
             this.resetFileLog(Svc.Prefs.get("log.appender.file.logOnError"),
                               LOG_PREFIX_ERROR);
             break;
         }
         break;
       case "weave:service:sync:finish":
-        this.resetFileLog(Svc.Prefs.get("log.appender.file.logOnSuccess"),
-                          LOG_PREFIX_SUCCESS);
+        if (Status.service == SYNC_FAILED_PARTIAL) {
+          this._log.debug("Some engines did not sync correctly.");
+          this.resetFileLog(Svc.Prefs.get("log.appender.file.logOnError"),
+                            LOG_PREFIX_ERROR);
+        } else {
+          this.resetFileLog(Svc.Prefs.get("log.appender.file.logOnSuccess"),
+                            LOG_PREFIX_SUCCESS);
+        }
         this._ignorableErrorCount = 0;
         break;
     }
   },
 
   /**
    * Generate a log file for the sync that just completed
    * and refresh the input & output streams.
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -381,17 +381,16 @@ WeaveSvc.prototype = {
 
     if (!this._checkCrypto()) {
       this.enabled = false;
       this._log.info("Could not load the Weave crypto component. Disabling " +
                       "Weave, since it will not work correctly.");
     }
 
     Svc.Obs.add("weave:service:setup-complete", this);
-    Svc.Obs.add("weave:engine:sync:applied", this);
     Svc.Prefs.observe("engine.", this);
 
     SyncScheduler.init();
 
     if (!this.enabled)
       this._log.info("Weave Sync disabled");
 
     // Create Weave identities (for logging in, and for encryption)
@@ -481,26 +480,16 @@ WeaveSvc.prototype = {
 
   observe: function WeaveSvc__observe(subject, topic, data) {
     switch (topic) {
       case "weave:service:setup-complete":
         let status = this._checkSetup();
         if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED)
             Svc.Obs.notify("weave:engine:start-tracking");
         break;
-      case "weave:engine:sync:applied":
-        if (subject.newFailed) {
-          // An engine isn't able to apply one or more incoming records.
-          // We don't fail hard on this, but it usually indicates a bug,
-          // so for now treat it as sync error (c.f. Service._syncEngine())
-          Status.engines = [data, ENGINE_APPLY_FAIL];
-          this._syncError = true;
-          this._log.debug(data + " failed to apply some records.");
-        }
-        break;
       case "nsPref:changed":
         if (this._ignorePrefObserver)
           return;
         let engine = data.slice((PREFS_BRANCH + "engine.").length);
         this._handleEngineStatusChanged(engine);
         break;
     }
   },
@@ -1396,29 +1385,28 @@ WeaveSvc.prototype = {
       // Upload meta/global if any engines changed anything
       let meta = Records.get(this.metaURL);
       if (meta.isNew || meta.changed) {
         new Resource(this.metaURL).put(meta);
         delete meta.isNew;
         delete meta.changed;
       }
 
-      if (this._syncError) {
-        throw "Some engines did not sync correctly";
-      } else {
+      // If there were no sync engine failures
+      if (Status.service != SYNC_FAILED_PARTIAL) {
         Svc.Prefs.set("lastSync", new Date().toString());
         Status.sync = SYNC_SUCCEEDED;
-        let syncTime = ((Date.now() - syncStartTime) / 1000).toFixed(2);
-        let dateStr = new Date().toLocaleFormat(LOG_DATE_FORMAT);
-        this._log.info("Sync completed successfully at " + dateStr
-                       + " after " + syncTime + " secs.");
       }
     } finally {
-      this._syncError = false;
       Svc.Prefs.reset("firstSync");
+
+      let syncTime = ((Date.now() - syncStartTime) / 1000).toFixed(2);
+      let dateStr = new Date().toLocaleFormat(LOG_DATE_FORMAT);
+      this._log.info("Sync completed at " + dateStr
+                     + " after " + syncTime + " secs.");
     }
   }))(),
 
 
   _updateEnabledEngines: function _updateEnabledEngines() {
     this._log.info("Updating enabled engines: " + SyncScheduler.numClients + " clients.");
     let meta = Records.get(this.metaURL);
     if (meta.isNew || !meta.payload.engines)
@@ -1490,29 +1478,22 @@ WeaveSvc.prototype = {
       return true;
     }
     catch(e) {
       // Maybe a 401, cluster update perhaps needed?
       if (e.status == 401) {
         // Log out and clear the cluster URL pref. That will make us perform
         // cluster detection and password check on next sync, which handles
         // both causes of 401s; in either case, we won't proceed with this
-        // sync, so return false, but kick off a sync for next time.
+        // sync so return false, but kick off a sync for next time.
         this.logout();
         Svc.Prefs.reset("clusterURL");
         Utils.nextTick(this.sync, this);
         return false;
       }
-
-      ErrorHandler.checkServerError(e);
-
-      Status.engines = [engine.name, e.failureCode || ENGINE_UNKNOWN_FAIL];
-
-      this._syncError = true;
-      this._log.debug(engine.name + " failed: " + Utils.exceptionStr(e));
       return true;
     }
   },
 
   /**
    * Silently fixes case issues.
    */
   syncKeyNeedsUpgrade: function syncKeyNeedsUpgrade() {
--- a/services/sync/tests/unit/test_errorhandler.js
+++ b/services/sync/tests/unit/test_errorhandler.js
@@ -4,21 +4,39 @@
 Cu.import("resource://services-sync/engines/clients.js");
 Cu.import("resource://services-sync/constants.js");
 Cu.import("resource://services-sync/policies.js");
 Cu.import("resource://services-sync/status.js");
 
 Svc.DefaultPrefs.set("registerEngines", "");
 Cu.import("resource://services-sync/service.js");
 
+const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"], true);
+const LOG_PREFIX_SUCCESS = "success-";
+const LOG_PREFIX_ERROR   = "error-";
+
+function CatapultEngine() {
+  SyncEngine.call(this, "Catapult");
+}
+CatapultEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  exception: null, // tests fill this in
+  sync: function sync() {
+    throw this.exception;
+  }
+};
+
+Engines.register(CatapultEngine);
+
 function run_test() {
   initTestLogging("Trace");
 
   Log4Moz.repository.getLogger("Sync.Service").level = Log4Moz.Level.Trace;
   Log4Moz.repository.getLogger("Sync.SyncScheduler").level = Log4Moz.Level.Trace;
+  Log4Moz.repository.getLogger("Sync.ErrorHandler").level = Log4Moz.Level.Trace;
 
   run_next_test();
 }
 
 function sync_httpd_setup() {
   let global = new ServerWBO("global", {
     syncID: Service.syncID,
     storageVersion: STORAGE_VERSION,
@@ -48,17 +66,17 @@ function setUp() {
   Service.username = "johndoe";
   Service.password = "ilovejane";
   Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/";
 
   generateNewKeys();
   let serverKeys = CollectionKeys.asWBO("crypto", "keys");
   serverKeys.encrypt(Service.syncKeyBundle);
-  return serverKeys.upload(Service.cryptoKeysURL);
+  return serverKeys.upload(Service.cryptoKeysURL).success;
 }
 
 add_test(function test_401_logout() {
   let server = sync_httpd_setup();
   setUp();
 
   // By calling sync, we ensure we're logged in.
   Service.sync();
@@ -119,8 +137,87 @@ add_test(function test_shouldIgnoreError
   Status.login = LOGIN_FAILED_LOGIN_REJECTED;
   Status.sync = NO_SYNC_NODE_FOUND;
 
   // Error not ignored.
   do_check_false(ErrorHandler.shouldIgnoreError());
 
   run_next_test();
 });
+
+add_test(function test_sync_engine_generic_fail() {
+  let server = sync_httpd_setup();
+
+  let engine = Engines.get("catapult");
+  engine.enabled = true;
+  engine.sync = function sync() {
+    Svc.Obs.notify("weave:engine:sync:error", "", "steam");
+  };
+
+  let log = Log4Moz.repository.getLogger("Sync.ErrorHandler");
+  Svc.Prefs.set("log.appender.file.logOnError", true);
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+    // Test Error log was written on SYNC_FAILED_PARTIAL.
+    let entries = logsdir.directoryEntries;
+    do_check_true(entries.hasMoreElements());
+    let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
+    do_check_eq(logfile.leafName.slice(0, LOG_PREFIX_ERROR.length),
+                LOG_PREFIX_ERROR);
+
+    Status.resetSync();
+    Service.startOver();
+
+    server.stop(run_next_test);
+  });
+
+  do_check_eq(Status.engines["steam"], undefined);
+
+  do_check_true(setUp());
+
+  Service.sync();
+
+  do_check_eq(Status.engines["steam"], ENGINE_UNKNOWN_FAIL);
+  do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
+});
+
+// This test should be the last one since it monkeypatches the engine object
+// and we should only have one engine object throughout the file (bug 629664).
+add_test(function test_engine_applyFailed() {
+  let server = sync_httpd_setup();
+
+  let engine = Engines.get("catapult");
+  engine.enabled = true;
+  delete engine.exception;
+  engine.sync = function sync() {
+    Svc.Obs.notify("weave:engine:sync:applied", {newFailed:1}, "steam");
+  };
+
+  let log = Log4Moz.repository.getLogger("Sync.ErrorHandler");
+  Svc.Prefs.set("log.appender.file.logOnError", true);
+
+  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+    // Test Error log was written on SYNC_FAILED_PARTIAL.
+    let entries = logsdir.directoryEntries;
+    do_check_true(entries.hasMoreElements());
+    let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
+    do_check_eq(logfile.leafName.slice(0, LOG_PREFIX_ERROR.length),
+                LOG_PREFIX_ERROR);
+
+    Status.resetSync();
+    Service.startOver();
+
+    server.stop(run_next_test);
+  });
+
+  do_check_eq(Status.engines["steam"], undefined);
+
+  do_check_true(setUp());
+
+  Service.sync();
+
+  do_check_eq(Status.engines["steam"], ENGINE_APPLY_FAIL);
+  do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
+});
--- a/services/sync/tests/unit/test_errorhandler_sync_checkServerError.js
+++ b/services/sync/tests/unit/test_errorhandler_sync_checkServerError.js
@@ -11,17 +11,17 @@ Cu.import("resource://services-sync/reco
 initTestLogging();
 
 function CatapultEngine() {
   SyncEngine.call(this, "Catapult");
 }
 CatapultEngine.prototype = {
   __proto__: SyncEngine.prototype,
   exception: null, // tests fill this in
-  sync: function sync() {
+  _sync: function _sync() {
     throw this.exception;
   }
 };
 
 function sync_httpd_setup() {
   let collectionsHelper = track_collections_helper();
   let upd = collectionsHelper.with_updated_collection;
   let collections = collectionsHelper.collections;
@@ -76,16 +76,17 @@ add_test(function test_backoff500() {
     do_check_false(Status.enforceBackoff);
 
     // Forcibly create and upload keys here -- otherwise we don't get to the 500!
     do_check_true(generateAndUploadKeys());
 
     Service.login();
     Service.sync();
     do_check_true(Status.enforceBackoff);
+    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
   } finally {
     Status.resetBackoff();
     Service.startOver();
   }
   server.stop(run_next_test);
 });
 
 add_test(function test_backoff503() {
@@ -109,16 +110,17 @@ add_test(function test_backoff503() {
 
     do_check_true(generateAndUploadKeys());
 
     Service.login();
     Service.sync();
 
     do_check_true(Status.enforceBackoff);
     do_check_eq(backoffInterval, BACKOFF);
+    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
   } finally {
     Status.resetBackoff();
     Service.startOver();
   }
   server.stop(run_next_test);
 });
 
 add_test(function test_overQuota() {
@@ -135,16 +137,17 @@ add_test(function test_overQuota() {
     do_check_eq(Status.sync, SYNC_SUCCEEDED);
 
     do_check_true(generateAndUploadKeys());
 
     Service.login();
     Service.sync();
 
     do_check_eq(Status.sync, OVER_QUOTA);
+    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
   } finally {
     Status.resetSync();
     Service.startOver();
   }
   server.stop(run_next_test);
 });
 
 add_test(function test_service_networkError() {
@@ -157,16 +160,17 @@ add_test(function test_service_networkEr
   try {
     do_check_eq(Status.sync, SYNC_SUCCEEDED);
 
     Service._loggedIn = true;
     Service.sync();
 
     do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
     do_check_eq(ErrorHandler._ignorableErrorCount, 1);
+    do_check_eq(Status.service, SYNC_FAILED);
   } finally {
     Status.resetSync();
     Service.startOver();
   }
   run_next_test();
 });
 
 add_test(function test_service_offline() {
@@ -178,16 +182,17 @@ add_test(function test_service_offline()
   try {
     do_check_eq(Status.sync, SYNC_SUCCEEDED);
 
     Service._loggedIn = true;
     Service.sync();
 
     do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
     do_check_eq(ErrorHandler._ignorableErrorCount, 0);
+    do_check_eq(Status.service, SYNC_FAILED);
   } finally {
     Status.resetSync();
     Service.startOver();
   }
   Services.io.offline = false;
   run_next_test();
 });
 
@@ -206,44 +211,44 @@ add_test(function test_service_reset_ign
 
     do_check_true(generateAndUploadKeys());
 
     Service.login();
     Service.sync();
 
     do_check_eq(Status.sync, SYNC_SUCCEEDED);
     do_check_eq(ErrorHandler._ignorableErrorCount, 0);
+    do_check_eq(Status.service, STATUS_OK);
   } finally {
     Status.resetSync();
     Service.startOver();
   }
   server.stop(run_next_test);
 });
 
 add_test(function test_engine_networkError() {
   _("Test: Network related exceptions from engine.sync() lead to the right status code.");
   setUp();
   let server = sync_httpd_setup();
-  ErrorHandler._ignorableErrorCount = 0;
 
   let engine = Engines.get("catapult");
   engine.enabled = true;
   engine.exception = Components.Exception("NS_ERROR_UNKNOWN_HOST",
                                           Cr.NS_ERROR_UNKNOWN_HOST);
 
   try {
     do_check_eq(Status.sync, SYNC_SUCCEEDED);
 
     do_check_true(generateAndUploadKeys());
 
     Service.login();
     Service.sync();
 
     do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
-    do_check_eq(ErrorHandler._ignorableErrorCount, 1);
+    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
   } finally {
     Status.resetSync();
     Service.startOver();
   }
   server.stop(run_next_test);
 });
 
 add_test(function test_resource_timeout() {
@@ -260,52 +265,20 @@ add_test(function test_resource_timeout(
     do_check_eq(Status.sync, SYNC_SUCCEEDED);
 
     do_check_true(generateAndUploadKeys());
 
     Service.login();
     Service.sync();
 
     do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
+    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
   } finally {
     Status.resetSync();
     Service.startOver();
   }
   server.stop(run_next_test);
 });
 
-
-// Slightly misplaced test as it doesn't actually test checkServerError,
-// but the observer for "weave:engine:sync:applied".
-// This test should be the last one since it monkeypatches the engine object
-// and we should only have one engine object throughout the file (bug 629664).
-add_test(function test_engine_applyFailed() {
-  setUp();
-  let server = sync_httpd_setup();
-
-  let engine = Engines.get("catapult");
-  engine.enabled = true;
-  delete engine.exception;
-  engine.sync = function sync() {
-    Svc.Obs.notify("weave:engine:sync:applied", {newFailed:1}, "steam");
-  };
-
-  try {
-    do_check_eq(Status.engines["steam"], undefined);
-
-    do_check_true(generateAndUploadKeys());
-
-    Service.login();
-    Service.sync();
-
-    do_check_eq(Status.engines["steam"], ENGINE_APPLY_FAIL);
-  } finally {
-    Status.resetSync();
-    Service.startOver();
-  }
-  server.stop(run_next_test);
-});
-
-
 function run_test() {
   Engines.register(CatapultEngine);
   run_next_test();
 }
--- a/services/sync/tests/unit/test_syncscheduler.js
+++ b/services/sync/tests/unit/test_syncscheduler.js
@@ -7,16 +7,29 @@ Cu.import("resource://services-sync/engi
 Cu.import("resource://services-sync/engines/clients.js");
 Cu.import("resource://services-sync/constants.js");
 Cu.import("resource://services-sync/policies.js");
 Cu.import("resource://services-sync/status.js");
 
 Svc.DefaultPrefs.set("registerEngines", "");
 Cu.import("resource://services-sync/service.js");
 
+function CatapultEngine() {
+  SyncEngine.call(this, "Catapult");
+}
+CatapultEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  exception: null, // tests fill this in
+  _sync: function _sync() {
+    throw this.exception;
+  }
+};
+
+Engines.register(CatapultEngine);
+
 function sync_httpd_setup() {
   let global = new ServerWBO("global", {
     syncID: Service.syncID,
     storageVersion: STORAGE_VERSION,
     engines: {clients: {version: Clients.version,
                         syncID: Clients.syncID}}
   });
   let clientsColl = new ServerCollection({}, true);
@@ -25,30 +38,31 @@ function sync_httpd_setup() {
   let collectionsHelper = track_collections_helper();
   let upd = collectionsHelper.with_updated_collection;
 
   return httpd_setup({
     "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()),
     "/1.1/johndoe/info/collections": collectionsHelper.handler,
     "/1.1/johndoe/storage/crypto/keys":
       upd("crypto", (new ServerWBO("keys")).handler()),
-    "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler())
+    "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()),
+    "/user/1.0/johndoe/node/weave": httpd_handler(200, "OK", "null")
   });
 }
 
 function setUp() {
   Service.username = "johndoe";
   Service.password = "ilovejane";
   Service.passphrase = "abcdeabcdeabcdeabcdeabcdea";
   Service.clusterURL = "http://localhost:8080/";
 
   generateNewKeys();
   let serverKeys = CollectionKeys.asWBO("crypto", "keys");
   serverKeys.encrypt(Service.syncKeyBundle);
-  return serverKeys.upload(Service.cryptoKeysURL);
+  return serverKeys.upload(Service.cryptoKeysURL).success;
 }
 
 function run_test() {
   initTestLogging("Trace");
 
   Log4Moz.repository.getLogger("Sync.Service").level = Log4Moz.Level.Trace;
   Log4Moz.repository.getLogger("Sync.SyncScheduler").level = Log4Moz.Level.Trace;
 
@@ -480,8 +494,87 @@ add_test(function test_idle_adjustSyncIn
   SyncScheduler.updateClientMode();
   SyncScheduler.observe(null, "idle", Svc.Prefs.get("scheduler.idleTime"));
   do_check_eq(SyncScheduler.idle, true);
   do_check_eq(SyncScheduler.syncInterval, SyncScheduler.idleInterval);
 
   SyncScheduler.setDefaults();
   run_next_test();
 });
+
+add_test(function test_no_sync_node() {
+  // Test when Status.sync == NO_SYNC_NODE_FOUND
+  // it is not overwritten on sync:finish
+  let server = sync_httpd_setup();
+  setUp();
+
+  Service.serverURL = "http://localhost:8080/";
+
+  Service.sync();
+  do_check_eq(Status.sync, NO_SYNC_NODE_FOUND);
+  do_check_eq(SyncScheduler.syncTimer.delay, NO_SYNC_NODE_INTERVAL);
+
+  // Clean up.
+  Service.startOver();
+  Status.resetSync();
+  server.stop(run_next_test);
+});
+
+add_test(function test_sync_failed_partial_500s() {
+  _("Test a 5xx status calls handleSyncError.");
+  SyncScheduler._syncErrors = MAX_ERROR_COUNT_BEFORE_BACKOFF;
+  let server = sync_httpd_setup();
+
+  let engine = Engines.get("catapult");
+  engine.enabled = true;
+  engine.exception = {status: 500};
+
+  do_check_eq(Status.sync, SYNC_SUCCEEDED);
+
+  do_check_true(setUp());
+
+  Service.sync();
+
+  do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
+
+  let maxInterval = SyncScheduler._syncErrors * (2 * MINIMUM_BACKOFF_INTERVAL);
+  do_check_eq(Status.backoffInterval, 0);
+  do_check_true(Status.enforceBackoff);
+  do_check_eq(SyncScheduler._syncErrors, 4);
+  do_check_true(SyncScheduler.nextSync <= (Date.now() + maxInterval));
+  do_check_true(SyncScheduler.syncTimer.delay <= maxInterval);
+
+  Status.resetSync();
+  Service.startOver();
+  server.stop(run_next_test);
+});
+
+add_test(function test_sync_failed_partial_400s() {
+  _("Test a non-5xx status doesn't call handleSyncError.");
+  SyncScheduler._syncErrors = MAX_ERROR_COUNT_BEFORE_BACKOFF;
+  let server = sync_httpd_setup();
+
+  let engine = Engines.get("catapult");
+  engine.enabled = true;
+  engine.exception = {status: 400};
+
+  // Have multiple devices for an active interval.
+  Clients._store.create({id: "foo", cleartext: "bar"});
+
+  do_check_eq(Status.sync, SYNC_SUCCEEDED);
+
+  do_check_true(setUp());
+
+  Service.sync();
+
+  do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
+  do_check_eq(SyncScheduler.syncInterval, SyncScheduler.activeInterval);
+
+  do_check_eq(Status.backoffInterval, 0);
+  do_check_false(Status.enforceBackoff);
+  do_check_eq(SyncScheduler._syncErrors, 0);
+  do_check_true(SyncScheduler.nextSync <= (Date.now() + SyncScheduler.activeInterval));
+  do_check_true(SyncScheduler.syncTimer.delay <= SyncScheduler.activeInterval);
+
+  Status.resetSync();
+  Service.startOver();
+  server.stop(run_next_test);
+});