Bug 1124428 - Add telemetry probes for FxA-related authentication issues. r=markh,vladan
authorKit Cambridge <kcambridge@mozilla.com>
Tue, 15 Sep 2015 18:18:04 -0700
changeset 295322 902358f27dd2f8d8ffabfeba7a088390c13813b8
parent 295321 ee2ddd22b6e55f39e9a47179f345df223ca3d692
child 295323 d5d0b5994149bc22ee38b1ae14acb69853a9646d
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmarkh, vladan
bugs1124428
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1124428 - Add telemetry probes for FxA-related authentication issues. r=markh,vladan
services/common/tokenserverclient.js
services/fxaccounts/FxAccounts.jsm
services/fxaccounts/FxAccountsClient.jsm
services/fxaccounts/tests/xpcshell/test_accounts.js
services/fxaccounts/tests/xpcshell/test_client.js
services/sync/modules-testing/rotaryengine.js
services/sync/modules-testing/utils.js
services/sync/modules/browserid_identity.js
services/sync/modules/engines.js
services/sync/modules/policies.js
services/sync/modules/resource.js
services/sync/modules/service.js
services/sync/modules/stages/cluster.js
services/sync/modules/stages/enginesync.js
services/sync/tests/unit/test_addons_store.js
services/sync/tests/unit/test_browserid_identity.js
services/sync/tests/unit/test_engine.js
services/sync/tests/unit/test_errorhandler.js
services/sync/tests/unit/test_service_sync_remoteSetup.js
services/sync/tests/unit/test_syncengine_sync.js
toolkit/components/telemetry/Histograms.json
--- a/services/common/tokenserverclient.js
+++ b/services/common/tokenserverclient.js
@@ -383,16 +383,21 @@ TokenServerClient.prototype = {
           error.cause = "conditions-required";
           error.urls = result.urls;
         }
       } else if (response.status == 404) {
         error.message = "Unknown service.";
         error.cause = "unknown-service";
       }
 
+      if (response.status == 401 || response.status == 403) {
+        Services.telemetry.getKeyedHistogramById(
+          "TOKENSERVER_AUTH_ERRORS").add(error.cause || "unknown");
+      }
+
       // A Retry-After header should theoretically only appear on a 503, but
       // we'll look for it on any error response.
       this._maybeNotifyBackoff(response, "retry-after");
 
       cb(error, null);
       return;
     }
 
--- a/services/fxaccounts/FxAccounts.jsm
+++ b/services/fxaccounts/FxAccounts.jsm
@@ -484,16 +484,17 @@ FxAccountsInternal.prototype = {
       let currentAccountState = this.currentAccountState = this.newAccountState(
         Cu.cloneInto(credentials, {}) // Pass a clone of the credentials object.
       );
       // This promise waits for storage, but not for verification.
       // We're telling the caller that this is durable now (although is that
       // really something we should commit to? Why not let the write happen in
       // the background? Already does for updateAccountData ;)
       return currentAccountState.promiseInitialized.then(() => {
+        Services.telemetry.getHistogramById("FXA_CONFIGURED").add(1);
         this.notifyObservers(ONLOGIN_NOTIFICATION);
         if (!this.isUserEmailVerified(credentials)) {
           this.startVerifiedCheck(credentials);
         }
       }).then(() => {
         return currentAccountState.resolve();
       });
     })
@@ -894,18 +895,21 @@ FxAccountsInternal.prototype = {
 
   /**
    * Setup for and if necessary do email verification polling.
    */
   loadAndPoll: function() {
     let currentState = this.currentAccountState;
     return currentState.getUserAccountData()
       .then(data => {
-        if (data && !this.isUserEmailVerified(data)) {
-          this.pollEmailStatus(currentState, data.sessionToken, "start");
+        if (data) {
+          Services.telemetry.getHistogramById("FXA_CONFIGURED").add(1);
+          if (!this.isUserEmailVerified(data)) {
+            this.pollEmailStatus(currentState, data.sessionToken, "start");
+          }
         }
         return data;
       });
   },
 
   startVerifiedCheck: function(data) {
     log.debug("startVerifiedCheck", data && data.verified);
     if (logPII) {
--- a/services/fxaccounts/FxAccountsClient.jsm
+++ b/services/fxaccounts/FxAccountsClient.jsm
@@ -13,16 +13,27 @@ Cu.import("resource://services-common/ut
 Cu.import("resource://services-common/hawkclient.js");
 Cu.import("resource://services-common/hawkrequest.js");
 Cu.import("resource://services-crypto/utils.js");
 Cu.import("resource://gre/modules/FxAccountsCommon.js");
 Cu.import("resource://gre/modules/Credentials.jsm");
 
 const HOST = Services.prefs.getCharPref("identity.fxaccounts.auth.uri");
 
+const STATUS_CODE_TO_OTHER_ERRORS_LABEL = {
+  400: 0,
+  401: 1,
+  403: 2,
+  410: 3,
+  411: 4,
+  413: 5,
+  429: 6,
+  500: 7,
+};
+
 this.FxAccountsClient = function(host = HOST) {
   this.host = host;
 
   // The FxA auth server expects requests to certain endpoints to be authorized
   // using Hawk.
   this.hawk = new HawkClient(host);
   this.hawk.observerPrefix = "FxA:hawk";
 
@@ -366,16 +377,40 @@ this.FxAccountsClient.prototype = {
           // Schedule clearing of cached-error-as-flag.
           CommonUtils.namedTimer(
             this._clearBackoff,
             error.retryAfter * 1000,
             this,
             "fxaBackoffTimer"
            );
         }
+        if (error.errno == ERRNO_UNVERIFIED_ACCOUNT) {
+          Services.telemetry.getKeyedHistogramById(
+            "FXA_UNVERIFIED_ACCOUNT_ERRORS").add(path);
+        } else if (isInvalidTokenError(error)) {
+          Services.telemetry.getKeyedHistogramById(
+            "FXA_HAWK_ERRORS").add(path);
+        } else if (error.code >= 500 || error.code in STATUS_CODE_TO_OTHER_ERRORS_LABEL) {
+          let label = STATUS_CODE_TO_OTHER_ERRORS_LABEL[
+            error.code >= 500 ? 500 : error.code];
+          Services.telemetry.getKeyedHistogramById(
+            "FXA_SERVER_ERRORS").add(path, label);
+        }
         deferred.reject(error);
       }
     );
 
     return deferred.promise;
   },
 };
 
+function isInvalidTokenError(error) {
+  if (error.code != 401) {
+    return false;
+  }
+  switch (error.errno) {
+    case ERRNO_INVALID_AUTH_TOKEN:
+    case ERRNO_INVALID_AUTH_TIMESTAMP:
+    case ERRNO_INVALID_AUTH_NONCE:
+      return true;
+  }
+  return false;
+}
--- a/services/fxaccounts/tests/xpcshell/test_accounts.js
+++ b/services/fxaccounts/tests/xpcshell/test_accounts.js
@@ -221,16 +221,19 @@ add_task(function test_get_signed_in_use
     kA: "beef",
     kB: "cafe",
     verified: true
   };
   let result = yield account.getSignedInUser();
   do_check_eq(result, null);
 
   yield account.setSignedInUser(credentials);
+  let histogram = Services.telemetry.getHistogramById("FXA_CONFIGURED");
+  do_check_eq(histogram.snapshot().sum, 1);
+  histogram.clear();
 
   result = yield account.getSignedInUser();
   do_check_eq(result.email, credentials.email);
   do_check_eq(result.assertion, credentials.assertion);
   do_check_eq(result.kB, credentials.kB);
 
   // Delete the memory cache and force the user
   // to be read and parsed from storage (e.g. disk via JSONStorage).
--- a/services/fxaccounts/tests/xpcshell/test_client.js
+++ b/services/fxaccounts/tests/xpcshell/test_client.js
@@ -601,16 +601,101 @@ add_task(function test_accountExists() {
     do_throw("Expected to catch an exception");
   } catch(unexpectedError) {
     do_check_eq(unexpectedError.code, 500);
   }
 
   yield deferredStop(server);
 });
 
+add_task(function* test_client_metrics() {
+  ["FXA_UNVERIFIED_ACCOUNT_ERRORS", "FXA_HAWK_ERRORS", "FXA_SERVER_ERRORS"].forEach(name => {
+    let histogram = Services.telemetry.getKeyedHistogramById(name);
+    histogram.clear();
+  });
+
+  function writeResp(response, msg) {
+    if (typeof msg === "object") {
+      msg = JSON.stringify(msg);
+    }
+    response.bodyOutputStream.write(msg, msg.length);
+  }
+  function assertSnapshot(name, key, message) {
+    let histogram = Services.telemetry.getKeyedHistogramById(name);
+    let snapshot = histogram.snapshot(key);
+    do_check_eq(snapshot.sum, 1, message);
+    histogram.clear();
+  }
+
+  let server = httpd_setup(
+    {
+      "/account/keys": function(request, response) {
+        response.setHeader("Content-Type", "application/json; charset=utf-8");
+        response.setStatusLine(request.httpVersion, 400, "Bad Request");
+        writeResp(response, {
+          error: "unverified account",
+          code: 400,
+          errno: 104,
+        });
+      },
+      "/session/destroy": function(request, response) {
+        response.setHeader("Content-Type", "application/json; charset=utf-8");
+        response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+        writeResp(response, {
+          error: "invalid authentication timestamp",
+          code: 401,
+          errno: 111,
+        });
+      },
+      "/recovery_email/status": function(request, response) {
+        response.setHeader("Content-Type", "application/json; charset=utf-8");
+        response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+        writeResp(response, {
+          error: " invalid request signature",
+          code: 401,
+          errno: 109,
+        });
+      },
+      "/recovery_email/resend_code": function(request, response) {
+        response.setHeader("Content-Type", "text/html");
+        response.setStatusLine(request.httpVersion, 504, "Sad Server");
+        writeResp(response, "<!doctype html><title>Simulated proxy error</title>");
+      },
+    }
+  );
+
+  let client = new FxAccountsClient(server.baseURI);
+
+  yield rejects(client.accountKeys(ACCOUNT_KEYS.keyFetch), function(err) {
+    return err.errno == 104;
+  });
+  assertSnapshot("FXA_UNVERIFIED_ACCOUNT_ERRORS", "/account/keys",
+    "Should report unverified account errors");
+
+  yield rejects(client.signOut(FAKE_SESSION_TOKEN), function(err) {
+    return err.errno == 111;
+  });
+  assertSnapshot("FXA_HAWK_ERRORS", "/session/destroy",
+    "Should report Hawk authentication errors");
+
+  yield rejects(client.recoveryEmailStatus(FAKE_SESSION_TOKEN), function(err) {
+    return err.errno == 109;
+  });
+  assertSnapshot("FXA_SERVER_ERRORS", "/recovery_email/status",
+    "Should report 400-class errors");
+
+  yield rejects(client.resendVerificationEmail(FAKE_SESSION_TOKEN), function(err) {
+    return err.code == 504;
+  });
+  assertSnapshot("FXA_SERVER_ERRORS", "/recovery_email/resend_code",
+    "Should report 500-class errors");
+
+  yield deferredStop(server);
+});
+
 add_task(function test_email_case() {
   let canonicalEmail = "greta.garbo@gmail.com";
   let clientEmail = "Greta.Garbo@gmail.COM";
   let attempts = 0;
 
   function writeResp(response, msg) {
     if (typeof msg === "object") {
       msg = JSON.stringify(msg);
--- a/services/sync/modules-testing/rotaryengine.js
+++ b/services/sync/modules-testing/rotaryengine.js
@@ -27,18 +27,18 @@ Cu.import("resource://services-sync/util
 this.RotaryRecord = function RotaryRecord(collection, id) {
   CryptoWrapper.call(this, collection, id);
 }
 RotaryRecord.prototype = {
   __proto__: CryptoWrapper.prototype
 };
 Utils.deferGetSet(RotaryRecord, "cleartext", ["denomination"]);
 
-this.RotaryStore = function RotaryStore(engine) {
-  Store.call(this, "Rotary", engine);
+this.RotaryStore = function RotaryStore(name, engine) {
+  Store.call(this, name, engine);
   this.items = {};
 }
 RotaryStore.prototype = {
   __proto__: Store.prototype,
 
   create: function create(record) {
     this.items[record.id] = record.denomination;
   },
@@ -83,18 +83,18 @@ RotaryStore.prototype = {
     return ids;
   },
 
   wipe: function wipe() {
     this.items = {};
   }
 };
 
-this.RotaryTracker = function RotaryTracker(engine) {
-  Tracker.call(this, "Rotary", engine);
+this.RotaryTracker = function RotaryTracker(name, engine) {
+  Tracker.call(this, name, engine);
 }
 RotaryTracker.prototype = {
   __proto__: Tracker.prototype
 };
 
 
 this.RotaryEngine = function RotaryEngine(service) {
   SyncEngine.call(this, "Rotary", service);
--- a/services/sync/modules-testing/utils.js
+++ b/services/sync/modules-testing/utils.js
@@ -13,32 +13,34 @@ this.EXPORTED_SYMBOLS = [
   "configureFxAccountIdentity",
   "configureIdentity",
   "SyncTestingInfrastructure",
   "waitForZeroTimer",
   "Promise", // from a module import
   "add_identity_test",
   "MockFxaStorageManager",
   "AccountState", // from a module import
+  "sumHistogram",
 ];
 
 const {utils: Cu} = Components;
 
 Cu.import("resource://services-sync/status.js");
 Cu.import("resource://services-sync/identity.js");
 Cu.import("resource://services-common/utils.js");
 Cu.import("resource://services-crypto/utils.js");
 Cu.import("resource://services-sync/util.js");
 Cu.import("resource://services-sync/browserid_identity.js");
 Cu.import("resource://testing-common/services/common/logging.js");
 Cu.import("resource://testing-common/services/sync/fakeservices.js");
 Cu.import("resource://gre/modules/FxAccounts.jsm");
 Cu.import("resource://gre/modules/FxAccountsClient.jsm");
 Cu.import("resource://gre/modules/FxAccountsCommon.js");
 Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
 
 // and grab non-exported stuff via a backstage pass.
 const {AccountState} = Cu.import("resource://gre/modules/FxAccounts.jsm", {});
 
 // A mock "storage manager" for FxAccounts that doesn't actually write anywhere.
 function MockFxaStorageManager() {
 }
 
@@ -312,8 +314,20 @@ this.add_identity_test = function(test, 
   test.add_task(function() {
     note("FxAccounts");
     let oldIdentity = Status._authManager;
     Status.__authManager = ns.Service.identity = new BrowserIDManager();
     yield testFunction();
     Status.__authManager = ns.Service.identity = oldIdentity;
   });
 }
+
+this.sumHistogram = function(name, options = {}) {
+  let histogram = options.key ? Services.telemetry.getKeyedHistogramById(name) :
+                  Services.telemetry.getHistogramById(name);
+  let snapshot = histogram.snapshot(options.key);
+  let sum = -Infinity;
+  if (snapshot) {
+    sum = snapshot.sum;
+  }
+  histogram.clear();
+  return sum;
+}
--- a/services/sync/modules/browserid_identity.js
+++ b/services/sync/modules/browserid_identity.js
@@ -470,33 +470,43 @@ this.BrowserIDManager.prototype = {
    * Verify the current auth state, unlocking the master-password if necessary.
    *
    * Returns a promise that resolves with the current auth state after
    * attempting to unlock.
    */
   unlockAndVerifyAuthState: function() {
     if (this._canFetchKeys()) {
       log.debug("unlockAndVerifyAuthState already has (or can fetch) sync keys");
+      Services.telemetry.getHistogramById("WEAVE_CAN_FETCH_KEYS").add(1);
       return Promise.resolve(STATUS_OK);
     }
     // so no keys - ensure MP unlocked.
     if (!Utils.ensureMPUnlocked()) {
       // user declined to unlock, so we don't know if they are stored there.
       log.debug("unlockAndVerifyAuthState: user declined to unlock master-password");
       return Promise.resolve(MASTER_PASSWORD_LOCKED);
     }
     // now we are unlocked we must re-fetch the user data as we may now have
     // the details that were previously locked away.
     return this._fxaService.getSignedInUser().then(
       accountData => {
         this._updateSignedInUser(accountData);
         // If we still can't get keys it probably means the user authenticated
         // without unlocking the MP or cleared the saved logins, so we've now
         // lost them - the user will need to reauth before continuing.
-        let result = this._canFetchKeys() ? STATUS_OK : LOGIN_FAILED_LOGIN_REJECTED;
+        let result;
+        if (this._canFetchKeys()) {
+          // A ternary would be more compact, but adding 0 to a flag histogram
+          // crashes the process with a C++ assertion error. See
+          // FlagHistogram::Accumulate in ipc/chromium/src/base/histogram.cc.
+          Services.telemetry.getHistogramById("WEAVE_CAN_FETCH_KEYS").add(1);
+          result = STATUS_OK;
+        } else {
+          result = LOGIN_FAILED_LOGIN_REJECTED;
+        }
         log.debug("unlockAndVerifyAuthState re-fetched credentials and is returning", result);
         return result;
       }
     );
   },
 
   /**
    * Do we have a non-null, not yet expired token for the user currently
@@ -617,16 +627,17 @@ this.BrowserIDManager.prototype = {
         // both tokenserverclient and hawkclient.
         // A tokenserver error thrown based on a bad response.
         if (err.response && err.response.status === 401) {
           err = new AuthenticationError(err);
         // A hawkclient error.
         } else if (err.code && err.code === 401) {
           err = new AuthenticationError(err);
         }
+        Services.telemetry.getHistogramById("WEAVE_FXA_KEY_FETCH_ERRORS").add();
 
         // TODO: write tests to make sure that different auth error cases are handled here
         // properly: auth error getting assertion, auth error getting token (invalid generation
         // and client-state error)
         if (err instanceof AuthenticationError) {
           this._log.error("Authentication error in _fetchTokenForUser", err);
           // set it to the "fatal" LOGIN_FAILED_LOGIN_REJECTED reason.
           this._authFailureReason = LOGIN_FAILED_LOGIN_REJECTED;
--- a/services/sync/modules/engines.js
+++ b/services/sync/modules/engines.js
@@ -305,16 +305,17 @@ Store.prototype = {
       } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
         // This kind of exception should have a 'cause' attribute, which is an
         // originating exception.
         // ex.cause will carry its stack with it when rethrown.
         throw ex.cause;
       } catch (ex if !Async.isShutdownException(ex)) {
         this._log.warn("Failed to apply incoming record " + record.id);
         this._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
+        this.engine._noteApplyFailure();
         failed.push(record.id);
       }
     };
     return failed;
   },
 
   /**
    * Apply a single record against the store.
@@ -1060,40 +1061,44 @@ SyncEngine.prototype = {
             case null:
               // Retry succeeded! No further handling.
               break;
             case SyncEngine.kRecoveryStrategy.retry:
               self._log.debug("Ignoring second retry suggestion.");
               // Fall through to error case.
             case SyncEngine.kRecoveryStrategy.error:
               self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
+              self._noteApplyFailure();
               failed.push(item.id);
               return;
             case SyncEngine.kRecoveryStrategy.ignore:
               self._log.debug("Ignoring record " + item.id +
                               " with bad HMAC: already handled.");
               return;
           }
         }
       } catch (ex) {
         self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
+        self._noteApplyFailure();
         failed.push(item.id);
         return;
       }
 
       let shouldApply;
       try {
         shouldApply = self._reconcile(item);
       } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
         self._log.warn("Reconciliation failed: aborting incoming processing.");
+        self._noteApplyFailure();
         failed.push(item.id);
         aborting = ex.cause;
       } catch (ex if !Async.isShutdownException(ex)) {
         self._log.warn("Failed to reconcile incoming record " + item.id);
         self._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
+        self._noteApplyFailure();
         failed.push(item.id);
         return;
       }
 
       if (shouldApply) {
         count.applied++;
         applyBatch.push(item);
       } else {
@@ -1189,27 +1194,43 @@ SyncEngine.prototype = {
       if (this.lastSync < this.lastModified) {
         this.lastSync = this.lastModified;
       }
     }
 
     // Apply remaining items.
     doApplyBatchAndPersistFailed.call(this);
 
-    count.newFailed = Utils.arraySub(this.previousFailed, failedInPreviousSync).length;
+    count.newFailed = this.previousFailed.reduce((count, engine) => {
+      if (failedInPreviousSync.indexOf(engine) == -1) {
+        count++;
+        this._noteApplyNewFailure();
+      }
+      return count;
+    }, 0);
     count.succeeded = Math.max(0, count.applied - count.failed);
     this._log.info(["Records:",
                     count.applied, "applied,",
                     count.succeeded, "successfully,",
                     count.failed, "failed to apply,",
                     count.newFailed, "newly failed to apply,",
                     count.reconciled, "reconciled."].join(" "));
     Observers.notify("weave:engine:sync:applied", count, this.name);
   },
 
+  _noteApplyFailure: function () {
+    Services.telemetry.getKeyedHistogramById(
+      "WEAVE_ENGINE_APPLY_FAILURES").add(this.name);
+  },
+
+  _noteApplyNewFailure: function () {
+    Services.telemetry.getKeyedHistogramById(
+      "WEAVE_ENGINE_APPLY_NEW_FAILURES").add(this.name);
+  },
+
   /**
    * Find a GUID of an item that is a duplicate of the incoming item but happens
    * to have a different GUID
    *
    * @return GUID of the similar item; falsy otherwise
    */
   _findDupe: function (item) {
     // By default, assume there's no dupe items for the engine
--- a/services/sync/modules/policies.js
+++ b/services/sync/modules/policies.js
@@ -591,20 +591,23 @@ ErrorHandler.prototype = {
           Status.engines = [data, ENGINE_APPLY_FAIL];
           this._log.debug(data + " failed to apply some records.");
         }
         break;
       case "weave:engine:sync:error":
         let exception = subject;  // exception thrown by engine's sync() method
         let engine_name = data;   // engine name that threw the exception
 
-        this.checkServerError(exception);
+        this.checkServerError(exception, "engines/" + engine_name);
 
         Status.engines = [engine_name, exception.failureCode || ENGINE_UNKNOWN_FAIL];
         this._log.debug(engine_name + " failed: " + Utils.exceptionStr(exception));
+
+        Services.telemetry.getKeyedHistogramById("WEAVE_ENGINE_SYNC_ERRORS")
+                          .add(engine_name);
         break;
       case "weave:service:login:error":
         this._log.error("Sync encountered a login error");
         this.resetFileLog();
 
         if (this.shouldReportError()) {
           this.notifyOnNextTick("weave:ui:login:error");
         } else {
@@ -841,17 +844,17 @@ ErrorHandler.prototype = {
   },
 
   /**
    * Handle HTTP response results or exceptions and set the appropriate
    * Status.* bits.
    *
    * This method also looks for "side-channel" warnings.
    */
-  checkServerError: function (resp) {
+  checkServerError: function (resp, cause) {
     switch (resp.status) {
       case 200:
       case 404:
       case 513:
         let xwa = resp.headers['x-weave-alert'];
 
         // Only process machine-readable alerts.
         if (!xwa || !xwa.startsWith("{")) {
@@ -871,16 +874,19 @@ ErrorHandler.prototype = {
 
       case 400:
         if (resp == RESPONSE_OVER_QUOTA) {
           Status.sync = OVER_QUOTA;
         }
         break;
 
       case 401:
+        Services.telemetry.getKeyedHistogramById(
+          "WEAVE_STORAGE_AUTH_ERRORS").add(cause);
+
         this.service.logout();
         this._log.info("Got 401 response; resetting clusterURL.");
         Svc.Prefs.reset("clusterURL");
 
         let delay = 0;
         if (Svc.Prefs.get("lastSyncReassigned")) {
           // We got a 401 in the middle of the previous sync, and we just got
           // another. Login must have succeeded in order for us to get here, so
--- a/services/sync/modules/resource.js
+++ b/services/sync/modules/resource.js
@@ -319,16 +319,17 @@ AsyncResource.prototype = {
       }
     } catch (ex) {
       this._log.debug("Caught exception " + CommonUtils.exceptionStr(ex) +
                       " visiting headers in _onComplete.");
       this._log.debug(CommonUtils.stackTrace(ex));
     }
 
     let ret     = new String(data);
+    ret.url     = channel.URI.spec;
     ret.status  = status;
     ret.success = success;
     ret.headers = headers;
 
     // Make a lazy getter to convert the json response into an object.
     // Note that this can cause a parse error to be thrown far away from the
     // actual fetch, so be warned!
     XPCOMUtils.defineLazyGetter(ret, "obj", function() {
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -553,22 +553,22 @@ Sync11Service.prototype = {
   _fetchInfo: function (url) {
     let infoURL = url || this.infoURL;
 
     this._log.trace("In _fetchInfo: " + infoURL);
     let info;
     try {
       info = this.resource(infoURL).get();
     } catch (ex) {
-      this.errorHandler.checkServerError(ex);
+      this.errorHandler.checkServerError(ex, "info/collections");
       throw ex;
     }
 
     // Always check for errors; this is also where we look for X-Weave-Alert.
-    this.errorHandler.checkServerError(info);
+    this.errorHandler.checkServerError(info, "info/collections");
     if (!info.success) {
       throw "Aborting sync: failed to get collections.";
     }
     return info;
   },
 
   verifyAndFetchSymmetricKeys: function verifyAndFetchSymmetricKeys(infoResponse) {
 
@@ -596,17 +596,17 @@ Sync11Service.prototype = {
     try {
       if (!infoResponse)
         infoResponse = this._fetchInfo();    // Will throw an exception on failure.
 
       // This only applies when the server is already at version 4.
       if (infoResponse.status != 200) {
         this._log.warn("info/collections returned non-200 response. Failing key fetch.");
         this.status.login = LOGIN_FAILED_SERVER_ERROR;
-        this.errorHandler.checkServerError(infoResponse);
+        this.errorHandler.checkServerError(infoResponse, "info/collections");
         return false;
       }
 
       let infoCollections = infoResponse.obj;
 
       this._log.info("Testing info/collections: " + JSON.stringify(infoCollections));
 
       if (this.collectionKeys.updateNeeded(infoCollections)) {
@@ -630,27 +630,29 @@ Sync11Service.prototype = {
               // On failure, ask to generate new keys and upload them.
               // Fall through to the behavior below.
               this._log.warn("Got 404 for crypto/keys, but 'crypto' in info/collections. Regenerating.");
               cryptoKeys = null;
             }
             else {
               // Some other problem.
               this.status.login = LOGIN_FAILED_SERVER_ERROR;
-              this.errorHandler.checkServerError(cryptoResp);
+              this.errorHandler.checkServerError(cryptoResp, "crypto/keys");
               this._log.warn("Got status " + cryptoResp.status + " fetching crypto keys.");
               return false;
             }
           }
           catch (ex) {
             this._log.warn("Got exception \"" + ex + "\" fetching cryptoKeys.");
             // TODO: Um, what exceptions might we get here? Should we re-throw any?
 
             // One kind of exception: HMAC failure.
             if (Utils.isHMACMismatch(ex)) {
+              Services.telemetry.getHistogramById(
+                "WEAVE_HMAC_ERRORS").add();
               this.status.login = LOGIN_FAILED_INVALID_PASSPHRASE;
               this.status.sync = CREDENTIALS_CHANGED;
             }
             else {
               // In the absence of further disambiguation or more precise
               // failure constants, just report failure.
               this.status.login = LOGIN_FAILED;
             }
@@ -679,17 +681,17 @@ Sync11Service.prototype = {
         // No update needed: we're good!
         return true;
       }
 
     } catch (ex) {
       // This means no keys are present, or there's a network error.
       this._log.debug("Failed to fetch and verify keys: "
                       + Utils.exceptionStr(ex));
-      this.errorHandler.checkServerError(ex);
+      this.errorHandler.checkServerError(ex, "crypto/keys");
       return false;
     }
   },
 
   verifyLogin: function verifyLogin(allow40XRecovery = true) {
     if (!this.identity.username) {
       this._log.warn("No username in verifyLogin.");
       this.status.login = LOGIN_FAILED_NO_USERNAME;
@@ -748,54 +750,56 @@ Sync11Service.prototype = {
           }
 
           this._log.warn("Remote setup failed.");
           // Remote setup must have failed.
           return false;
 
         case 401:
           this._log.warn("401: login failed.");
+          Services.telemetry.getKeyedHistogramById(
+            "WEAVE_STORAGE_AUTH_ERRORS").add("info/collections");
           // Fall through to the 404 case.
 
         case 404:
           // Check that we're verifying with the correct cluster
           if (allow40XRecovery && this._clusterManager.setCluster()) {
             return this.verifyLogin(false);
           }
 
           // We must have the right cluster, but the server doesn't expect us
           this.status.login = LOGIN_FAILED_LOGIN_REJECTED;
           return false;
 
         default:
           // Server didn't respond with something that we expected
           this.status.login = LOGIN_FAILED_SERVER_ERROR;
-          this.errorHandler.checkServerError(test);
+          this.errorHandler.checkServerError(test, "info/collections");
           return false;
       }
     } catch (ex) {
       // Must have failed on some network issue
       this._log.debug("verifyLogin failed: " + Utils.exceptionStr(ex));
       this.status.login = LOGIN_FAILED_NETWORK_ERROR;
-      this.errorHandler.checkServerError(ex);
+      this.errorHandler.checkServerError(ex, "info/collections");
       return false;
     }
   },
 
   generateNewSymmetricKeys: function generateNewSymmetricKeys() {
     this._log.info("Generating new keys WBO...");
     let wbo = this.collectionKeys.generateNewKeysWBO();
     this._log.info("Encrypting new key bundle.");
     wbo.encrypt(this.identity.syncKeyBundle);
 
     this._log.info("Uploading...");
     let uploadRes = wbo.upload(this.resource(this.cryptoKeysURL));
     if (uploadRes.status != 200) {
       this._log.warn("Got status " + uploadRes.status + " uploading new keys. What to do? Throw!");
-      this.errorHandler.checkServerError(uploadRes);
+      this.errorHandler.checkServerError(uploadRes, "crypto/keys");
       throw new Error("Unable to upload symmetric keys.");
     }
     this._log.info("Got status " + uploadRes.status + " uploading keys.");
     let serverModified = uploadRes.obj;   // Modified timestamp according to server.
     this._log.debug("Server reports crypto modified: " + serverModified);
 
     // Now verify that info/collections shows them!
     this._log.debug("Verifying server collection records.");
@@ -1090,32 +1094,34 @@ Sync11Service.prototype = {
 
       // ... fetch the current record from the server, and COPY THE FLAGS.
       let newMeta = this.recordManager.get(this.metaURL);
 
       // If we got a 401, we do not want to create a new meta/global - we
       // should be able to get the existing meta after we get a new node.
       if (this.recordManager.response.status == 401) {
         this._log.debug("Fetching meta/global record on the server returned 401.");
-        this.errorHandler.checkServerError(this.recordManager.response);
+        this.errorHandler.checkServerError(this.recordManager.response, "meta/global");
         return false;
       }
 
       if (!this.recordManager.response.success || !newMeta) {
         this._log.debug("No meta/global record on the server. Creating one.");
         newMeta = new WBORecord("meta", "global");
         newMeta.payload.syncID = this.syncID;
         newMeta.payload.storageVersion = STORAGE_VERSION;
         newMeta.payload.declined = this.engineManager.getDeclined();
 
         newMeta.isNew = true;
 
         this.recordManager.set(this.metaURL, newMeta);
-        if (!newMeta.upload(this.resource(this.metaURL)).success) {
+        let uploadRes = newMeta.upload(this.resource(this.metaURL));
+        if (!uploadRes.success) {
           this._log.warn("Unable to upload new meta/global. Failing remote setup.");
+          this.errorHandler.checkServerError(uploadRes, "meta/global");
           return false;
         }
       } else {
         // If newMeta, then it stands to reason that meta != null.
         newMeta.isNew   = meta.isNew;
         newMeta.changed = meta.changed;
       }
 
@@ -1136,17 +1142,17 @@ Sync11Service.prototype = {
         STORAGE_VERSION > parseFloat(remoteVersion)) {
 
       this._log.info("One of: no meta, no meta storageVersion, or no meta syncID. Fresh start needed.");
 
       // abort the server wipe if the GET status was anything other than 404 or 200
       let status = this.recordManager.response.status;
       if (status != 200 && status != 404) {
         this.status.sync = METARECORD_DOWNLOAD_FAIL;
-        this.errorHandler.checkServerError(this.recordManager.response);
+        this.errorHandler.checkServerError(this.recordManager.response, "meta/global");
         this._log.warn("Unknown error while downloading metadata record. " +
                        "Aborting sync.");
         return false;
       }
 
       if (!meta)
         this._log.info("No metadata record, server wipe needed");
       if (meta && !meta.payload.syncID)
@@ -1612,17 +1618,17 @@ Sync11Service.prototype = {
       // Tell the remote machines to wipe themselves.
       else {
         this.clientsEngine.sendCommand("wipeAll", []);
       }
 
       // Make sure the changed clients get updated.
       this.clientsEngine.sync();
     } catch (ex) {
-      this.errorHandler.checkServerError(ex);
+      this.errorHandler.checkServerError(ex, "clients");
       throw ex;
     }
   },
 
   /**
    * Reset local service information like logs, sync times, caches.
    */
   resetService: function resetService() {
--- a/services/sync/modules/stages/cluster.js
+++ b/services/sync/modules/stages/cluster.js
@@ -51,24 +51,24 @@ ClusterManager.prototype = {
         case 0:
         case 200:
           if (node == "null") {
             node = null;
           }
           this._log.trace("_findCluster successfully returning " + node);
           return node;
         default:
-          this.service.errorHandler.checkServerError(node);
+          this.service.errorHandler.checkServerError(node, "node/weave");
           fail = "Unexpected response code: " + node.status;
           break;
       }
     } catch (e) {
       this._log.debug("Network error on findCluster");
       this.service.status.login = LOGIN_FAILED_NETWORK_ERROR;
-      this.service.errorHandler.checkServerError(e);
+      this.service.errorHandler.checkServerError(e, "node/weave");
       fail = e;
     }
     throw fail;
   },
 
   /**
    * Determine the cluster for the current user and update state.
    */
--- a/services/sync/modules/stages/enginesync.js
+++ b/services/sync/modules/stages/enginesync.js
@@ -133,17 +133,17 @@ EngineSynchronizer.prototype = {
     }
 
     // Update engines because it might change what we sync.
     try {
       this._updateEnabledEngines();
     } catch (ex) {
       this._log.debug("Updating enabled engines failed: " +
                       Utils.exceptionStr(ex));
-      this.service.errorHandler.checkServerError(ex);
+      this.service.errorHandler.checkServerError(ex, "meta/global");
       this.onComplete(ex);
       return;
     }
 
     try {
       for (let engine of engineManager.getEnabled()) {
         // If there's any problems with syncing the engine, report the failure
         if (!(this._syncEngine(engine)) || this.service.status.enforceBackoff) {
--- a/services/sync/tests/unit/test_addons_store.js
+++ b/services/sync/tests/unit/test_addons_store.js
@@ -3,16 +3,17 @@
 
 "use strict";
 
 Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://services-sync/addonutils.js");
 Cu.import("resource://services-sync/engines/addons.js");
 Cu.import("resource://services-sync/service.js");
 Cu.import("resource://services-sync/util.js");
+Cu.import("resource://testing-common/services/sync/utils.js");
 
 const HTTP_PORT = 8888;
 
 var prefs = new Preferences();
 
 prefs.set("extensions.getAddons.get.url", "http://localhost:8888/search/guid:%IDS%");
 loadAddonTestFunctions();
 startupManager();
@@ -389,16 +390,17 @@ add_test(function test_create_missing_se
   // The handler for this ID is not installed, so a search should 404.
   const id = "missing@tests.mozilla.org";
   let guid = Utils.makeGUID();
   let record = createRecordForThisApp(guid, id, true, false);
 
   let failed = store.applyIncomingBatch([record]);
   do_check_eq(1, failed.length);
   do_check_eq(guid, failed[0]);
+  do_check_eq(sumHistogram("WEAVE_ENGINE_APPLY_FAILURES", { key: "addons" }), 1);
 
   let addon = getAddonFromAddonManagerByID(id);
   do_check_eq(null, addon);
 
   server.stop(run_next_test);
 });
 
 add_test(function test_create_bad_install() {
@@ -409,16 +411,17 @@ add_test(function test_create_bad_instal
   // The handler returns a search result but the XPI will 404.
   const id = "missing-xpi@tests.mozilla.org";
   let guid = Utils.makeGUID();
   let record = createRecordForThisApp(guid, id, true, false);
 
   let failed = store.applyIncomingBatch([record]);
   do_check_eq(1, failed.length);
   do_check_eq(guid, failed[0]);
+  do_check_eq(sumHistogram("WEAVE_ENGINE_APPLY_FAILURES", { key: "addons" }), 1);
 
   let addon = getAddonFromAddonManagerByID(id);
   do_check_eq(null, addon);
 
   server.stop(run_next_test);
 });
 
 add_test(function test_wipe() {
--- a/services/sync/tests/unit/test_browserid_identity.js
+++ b/services/sync/tests/unit/test_browserid_identity.js
@@ -405,31 +405,37 @@ add_task(function test_getTokenErrors() 
   });
   let browseridManager = Service.identity;
 
   yield browseridManager.initializeWithCurrentIdentity();
   yield Assert.rejects(browseridManager.whenReadyToAuthenticate.promise,
                        "should reject due to 401");
   Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED, "login was rejected");
 
+  let keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch error for rejected logins");
+
   // XXX - other interesting responses to return?
 
   // And for good measure, some totally "unexpected" errors - we generally
   // assume these problems are going to magically go away at some point.
   _("Arrange for an empty body with a 200 response - should reflect a network error.");
   initializeIdentityWithTokenServerResponse({
     status: 200,
     headers: [],
     body: "",
   });
   browseridManager = Service.identity;
   yield browseridManager.initializeWithCurrentIdentity();
   yield Assert.rejects(browseridManager.whenReadyToAuthenticate.promise,
                        "should reject due to non-JSON response");
   Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login state is LOGIN_FAILED_NETWORK_ERROR");
+
+  keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch errors for invalid responses");
 });
 
 add_task(function test_getTokenErrorWithRetry() {
   _("tokenserver sends an observer notification on various backoff headers.");
 
   // Set Sync's backoffInterval to zero - after we simulated the backoff header
   // it should reflect the value we sent.
   Status.backoffInterval = 0;
@@ -446,32 +452,38 @@ add_task(function test_getTokenErrorWith
   yield Assert.rejects(browseridManager.whenReadyToAuthenticate.promise,
                        "should reject due to 503");
 
   // The observer should have fired - check it got the value in the response.
   Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login was rejected");
   // Sync will have the value in ms with some slop - so check it is at least that.
   Assert.ok(Status.backoffInterval >= 100000);
 
+  let keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch errors for 503 from FxA");
+
   _("Arrange for a 200 with an X-Backoff header.");
   Status.backoffInterval = 0;
   initializeIdentityWithTokenServerResponse({
     status: 503,
     headers: {"content-type": "application/json",
               "x-backoff": "200"},
     body: JSON.stringify({}),
   });
   browseridManager = Service.identity;
 
   yield browseridManager.initializeWithCurrentIdentity();
   yield Assert.rejects(browseridManager.whenReadyToAuthenticate.promise,
                        "should reject due to no token in response");
 
   // The observer should have fired - check it got the value in the response.
   Assert.ok(Status.backoffInterval >= 200000);
+
+  keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch errors for backoff response from FxA");
 });
 
 add_task(function test_getKeysErrorWithBackoff() {
   _("Auth server (via hawk) sends an observer notification on backoff headers.");
 
   // Set Sync's backoffInterval to zero - after we simulated the backoff header
   // it should reflect the value we sent.
   Status.backoffInterval = 0;
@@ -496,16 +508,19 @@ add_task(function test_getKeysErrorWithB
   let browseridManager = Service.identity;
   yield Assert.rejects(browseridManager.whenReadyToAuthenticate.promise,
                        "should reject due to 503");
 
   // The observer should have fired - check it got the value in the response.
   Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login was rejected");
   // Sync will have the value in ms with some slop - so check it is at least that.
   Assert.ok(Status.backoffInterval >= 100000);
+
+  let keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch errors for 503 from FxA");
 });
 
 add_task(function test_getKeysErrorWithRetry() {
   _("Auth server (via hawk) sends an observer notification on retry headers.");
 
   // Set Sync's backoffInterval to zero - after we simulated the backoff header
   // it should reflect the value we sent.
   Status.backoffInterval = 0;
@@ -530,16 +545,19 @@ add_task(function test_getKeysErrorWithR
   let browseridManager = Service.identity;
   yield Assert.rejects(browseridManager.whenReadyToAuthenticate.promise,
                        "should reject due to 503");
 
   // The observer should have fired - check it got the value in the response.
   Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login was rejected");
   // Sync will have the value in ms with some slop - so check it is at least that.
   Assert.ok(Status.backoffInterval >= 100000);
+
+  let keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch errors for 503 from FxA");
 });
 
 add_task(function test_getHAWKErrors() {
   _("BrowserIDManager correctly handles various HAWK failures.");
 
   _("Arrange for a 401 - Sync should reflect an auth error.");
   let config = makeIdentityConfig();
   yield initializeIdentityWithHAWKResponseFactory(config, function(method, data, uri) {
@@ -548,31 +566,37 @@ add_task(function test_getHAWKErrors() {
     return {
       status: 401,
       headers: {"content-type": "application/json"},
       body: JSON.stringify({}),
     }
   });
   Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED, "login was rejected");
 
+  let keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch errors for 401 from FxA");
+
   // XXX - other interesting responses to return?
 
   // And for good measure, some totally "unexpected" errors - we generally
   // assume these problems are going to magically go away at some point.
   _("Arrange for an empty body with a 200 response - should reflect a network error.");
   yield initializeIdentityWithHAWKResponseFactory(config, function(method, data, uri) {
     Assert.equal(method, "post");
     Assert.equal(uri, "http://mockedserver:9999/certificate/sign")
     return {
       status: 200,
       headers: [],
       body: "",
     }
   });
   Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "login state is LOGIN_FAILED_NETWORK_ERROR");
+
+  keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch errors for invalid response from FxA");
 });
 
 add_task(function test_getGetKeysFailing401() {
   _("BrowserIDManager correctly handles 401 responses fetching keys.");
 
   _("Arrange for a 401 - Sync should reflect an auth error.");
   let config = makeIdentityConfig();
   // We want no kA or kB so we attempt to fetch them.
@@ -584,16 +608,19 @@ add_task(function test_getGetKeysFailing
     Assert.equal(uri, "http://mockedserver:9999/account/keys")
     return {
       status: 401,
       headers: {"content-type": "application/json"},
       body: "{}",
     }
   });
   Assert.equal(Status.login, LOGIN_FAILED_LOGIN_REJECTED, "login was rejected");
+
+  let keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch errors for 401 from FxA");
 });
 
 add_task(function test_getGetKeysFailing503() {
   _("BrowserIDManager correctly handles 5XX responses fetching keys.");
 
   _("Arrange for a 503 - Sync should reflect a network error.");
   let config = makeIdentityConfig();
   // We want no kA or kB so we attempt to fetch them.
@@ -605,16 +632,19 @@ add_task(function test_getGetKeysFailing
     Assert.equal(uri, "http://mockedserver:9999/account/keys")
     return {
       status: 503,
       headers: {"content-type": "application/json"},
       body: "{}",
     }
   });
   Assert.equal(Status.login, LOGIN_FAILED_NETWORK_ERROR, "state reflects network error");
+
+  let keyFetchErrorCount = sumHistogram("WEAVE_FXA_KEY_FETCH_ERRORS");
+  Assert.equal(keyFetchErrorCount, 1, "Should record key fetch errors for 503 from FxA");
 });
 
 add_task(function test_getKeysMissing() {
   _("BrowserIDManager correctly handles getKeys succeeding but not returning keys.");
 
   let browseridManager = new BrowserIDManager();
   let identityConfig = makeIdentityConfig();
   // our mock identity config already has kA and kB - remove them or we never
@@ -661,16 +691,54 @@ add_task(function test_getKeysMissing() 
     yield browseridManager.whenReadyToAuthenticate.promise;
   } catch (e) {
     ex = e;
   }
 
   Assert.ok(ex.message.indexOf("missing kA or kB") >= 0);
 });
 
+add_task(function* test_signedInUserMissing() {
+  _("BrowserIDManager detects getSignedInUser returning incomplete account data");
+
+  let browseridManager = new BrowserIDManager();
+  let config = makeIdentityConfig();
+  // Delete stored keys and the key fetch token.
+  delete identityConfig.fxaccount.user.kA;
+  delete identityConfig.fxaccount.user.kB;
+  delete identityConfig.fxaccount.user.keyFetchToken;
+
+  configureFxAccountIdentity(browseridManager, identityConfig);
+
+  let fxa = new FxAccounts({
+    fetchAndUnwrapKeys: function () {
+      return Promise.resolve({});
+    },
+    fxAccountsClient: new MockFxAccountsClient(),
+    newAccountState(credentials) {
+      // We only expect this to be called with null indicating the (mock)
+      // storage should be read.
+      if (credentials) {
+        throw new Error("Not expecting to have credentials passed");
+      }
+      let storageManager = new MockFxaStorageManager();
+      storageManager.initialize(identityConfig.fxaccount.user);
+      return new AccountState(storageManager);
+    },
+  });
+
+  browseridManager._fxaService = fxa;
+
+  let status = yield browseridManager.unlockAndVerifyAuthState();
+  Assert.equal(status, LOGIN_FAILED_LOGIN_REJECTED);
+
+  let canFetchKeysCount = sumHistogram("WEAVE_CAN_FETCH_KEYS");
+  Assert.equal(canFetchKeysCount, 0);
+});
+
 // End of tests
 // Utility functions follow
 
 // Create a new browserid_identity object and initialize it with a
 // hawk mock that simulates HTTP responses.
 // The callback function will be called each time the mocked hawk server wants
 // to make a request.  The result of the callback should be the mock response
 // object that will be returned to hawk.
--- a/services/sync/tests/unit/test_engine.js
+++ b/services/sync/tests/unit/test_engine.js
@@ -20,18 +20,18 @@ SteamStore.prototype = {
 
 function SteamTracker(name, engine) {
   Tracker.call(this, name || "Steam", engine);
 }
 SteamTracker.prototype = {
   __proto__: Tracker.prototype
 };
 
-function SteamEngine(service) {
-  Engine.call(this, "Steam", service);
+function SteamEngine(name, service) {
+  Engine.call(this, name, service);
   this.wasReset = false;
   this.wasSynced = false;
 }
 SteamEngine.prototype = {
   __proto__: Engine.prototype,
   _storeObj: SteamStore,
   _trackerObj: SteamTracker,
 
@@ -64,27 +64,27 @@ Observers.add("weave:engine:sync:start",
 Observers.add("weave:engine:sync:finish", engineObserver);
 
 function run_test() {
   run_next_test();
 }
 
 add_test(function test_members() {
   _("Engine object members");
-  let engine = new SteamEngine(Service);
+  let engine = new SteamEngine("Steam", Service);
   do_check_eq(engine.Name, "Steam");
   do_check_eq(engine.prefName, "steam");
   do_check_true(engine._store instanceof SteamStore);
   do_check_true(engine._tracker instanceof SteamTracker);
   run_next_test();
 });
 
 add_test(function test_score() {
   _("Engine.score corresponds to tracker.score and is readonly");
-  let engine = new SteamEngine(Service);
+  let engine = new SteamEngine("Steam", Service);
   do_check_eq(engine.score, 0);
   engine._tracker.score += 5;
   do_check_eq(engine.score, 5);
 
   try {
     engine.score = 10;
   } catch(ex) {
     // Setting an attribute that has a getter produces an error in
@@ -92,47 +92,47 @@ add_test(function test_score() {
     // the attribute's value won't change.
   }
   do_check_eq(engine.score, 5);
   run_next_test();
 });
 
 add_test(function test_resetClient() {
   _("Engine.resetClient calls _resetClient");
-  let engine = new SteamEngine(Service);
+  let engine = new SteamEngine("Steam", Service);
   do_check_false(engine.wasReset);
 
   engine.resetClient();
   do_check_true(engine.wasReset);
   do_check_eq(engineObserver.topics[0], "weave:engine:reset-client:start");
   do_check_eq(engineObserver.topics[1], "weave:engine:reset-client:finish");
 
   engine.wasReset = false;
   engineObserver.reset();
   run_next_test();
 });
 
 add_test(function test_invalidChangedIDs() {
   _("Test that invalid changed IDs on disk don't end up live.");
-  let engine = new SteamEngine(Service);
+  let engine = new SteamEngine("Steam", Service);
   let tracker = engine._tracker;
   tracker.changedIDs = 5;
   tracker.saveChangedIDs(function onSaved() {
       tracker.changedIDs = {placeholder: true};
       tracker.loadChangedIDs(function onLoaded(json) {
         do_check_null(json);
         do_check_true(tracker.changedIDs.placeholder);
         run_next_test();
       });
     });
 });
 
 add_test(function test_wipeClient() {
   _("Engine.wipeClient calls resetClient, wipes store, clears changed IDs");
-  let engine = new SteamEngine(Service);
+  let engine = new SteamEngine("Steam", Service);
   do_check_false(engine.wasReset);
   do_check_false(engine._store.wasWiped);
   do_check_true(engine._tracker.addChangedID("a-changed-id"));
   do_check_true("a-changed-id" in engine._tracker.changedIDs);
 
   engine.wipeClient();
   do_check_true(engine.wasReset);
   do_check_true(engine._store.wasWiped);
@@ -145,32 +145,32 @@ add_test(function test_wipeClient() {
   engine.wasReset = false;
   engine._store.wasWiped = false;
   engineObserver.reset();
   run_next_test();
 });
 
 add_test(function test_enabled() {
   _("Engine.enabled corresponds to preference");
-  let engine = new SteamEngine(Service);
+  let engine = new SteamEngine("Steam", Service);
   try {
     do_check_false(engine.enabled);
     Svc.Prefs.set("engine.steam", true);
     do_check_true(engine.enabled);
 
     engine.enabled = false;
     do_check_false(Svc.Prefs.get("engine.steam"));
     run_next_test();
   } finally {
     Svc.Prefs.resetBranch("");
   }
 });
 
 add_test(function test_sync() {
-  let engine = new SteamEngine(Service);
+  let engine = new SteamEngine("Steam", Service);
   try {
     _("Engine.sync doesn't call _sync if it's not enabled");
     do_check_false(engine.enabled);
     do_check_false(engine.wasSynced);
     engine.sync();
     do_check_false(engine.wasSynced);
 
     _("Engine.sync calls _sync if it's enabled");
@@ -184,17 +184,17 @@ add_test(function test_sync() {
     Svc.Prefs.resetBranch("");
     engine.wasSynced = false;
     engineObserver.reset();
   }
 });
 
 add_test(function test_disabled_no_track() {
   _("When an engine is disabled, its tracker is not tracking.");
-  let engine = new SteamEngine(Service);
+  let engine = new SteamEngine("Steam", Service);
   let tracker = engine._tracker;
   do_check_eq(engine, tracker.engine);
 
   do_check_false(engine.enabled);
   do_check_false(tracker._isTracking);
   do_check_empty(tracker.changedIDs);
 
   do_check_false(tracker.engineIsEnabled());
--- a/services/sync/tests/unit/test_errorhandler.js
+++ b/services/sync/tests/unit/test_errorhandler.js
@@ -176,16 +176,19 @@ add_identity_test(this, function test_40
     _("Got weave:service:sync:error in first sync.");
     Svc.Obs.remove("weave:service:sync:error", onSyncError);
 
     // Wait for the automatic next sync.
     function onLoginError() {
       _("Got weave:service:login:error in second sync.");
       Svc.Obs.remove("weave:service:login:error", onLoginError);
 
+      let errorCount = sumHistogram("WEAVE_STORAGE_AUTH_ERRORS", { key: "info/collections" });
+      do_check_eq(errorCount, 2);
+
       do_check_eq(Status.login, LOGIN_FAILED_LOGIN_REJECTED);
       do_check_false(Service.isLoggedIn);
 
       // Clean up.
       Utils.nextTick(function () {
         Service.startOver();
         server.stop(deferred.resolve);
       });
@@ -1796,16 +1799,20 @@ add_task(function test_sync_engine_gener
 
       // Test Error log was written on SYNC_FAILED_PARTIAL.
       let entries = logsdir.directoryEntries;
       do_check_true(entries.hasMoreElements());
       let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
       do_check_true(logfile.leafName.startsWith("error-sync-"), logfile.leafName);
 
       clean();
+
+      let syncErrors = sumHistogram("WEAVE_ENGINE_SYNC_ERRORS", { key: "catapult" });
+      do_check_true(syncErrors, 1);
+
       server.stop(deferred.resolve);
     });
   });
 
   do_check_true(yield setUp(server));
   Service.sync();
   yield deferred.promise;
 });
--- a/services/sync/tests/unit/test_service_sync_remoteSetup.js
+++ b/services/sync/tests/unit/test_service_sync_remoteSetup.js
@@ -156,13 +156,16 @@ function run_test() {
     b.generateRandom();
     collections.crypto = keys.modified = 100 + (Date.now()/1000);  // Future modification time.
     keys.encrypt(b);
     keys.upload(Service.resource(Service.cryptoKeysURL));
 
     do_check_false(Service.verifyAndFetchSymmetricKeys());
     do_check_eq(Service.status.login, LOGIN_FAILED_INVALID_PASSPHRASE);
 
+    let hmacErrors = sumHistogram("WEAVE_HMAC_ERRORS");
+    do_check_eq(hmacErrors, 1);
+
   } finally {
     Svc.Prefs.resetBranch("");
     server.stop(do_test_finished);
   }
 }
--- a/services/sync/tests/unit/test_syncengine_sync.js
+++ b/services/sync/tests/unit/test_syncengine_sync.js
@@ -847,16 +847,17 @@ add_test(function test_processIncoming_a
     engine._processIncoming();
 
     // Records have been applied and the expected failures have failed.
     do_check_attribute_count(engine._store.items, APPLY_BATCH_SIZE - 1 - 2);
     do_check_eq(engine.toFetch.length, 0);
     do_check_eq(engine.previousFailed.length, 2);
     do_check_eq(engine.previousFailed[0], "record-no-0");
     do_check_eq(engine.previousFailed[1], "record-no-8");
+    do_check_eq(sumHistogram("WEAVE_ENGINE_APPLY_NEW_FAILURES", { key: "rotary" }), 2);
 
   } finally {
     cleanAndGo(server);
   }
 });
 
 
 add_test(function test_processIncoming_applyIncomingBatchSize_multiple() {
@@ -900,16 +901,17 @@ add_test(function test_processIncoming_a
     do_check_empty(engine._store.items);
 
     engine._syncStartup();
     engine._processIncoming();
 
     // Records have been applied in 3 batches.
     do_check_eq(batchCalls, 3);
     do_check_attribute_count(engine._store.items, APPLY_BATCH_SIZE * 3);
+    do_check_eq(sumHistogram("WEAVE_ENGINE_APPLY_NEW_FAILURES", { key: "rotary" }), 3);
 
   } finally {
     cleanAndGo(server);
   }
 });
 
 
 add_test(function test_processIncoming_notify_count() {
@@ -975,30 +977,35 @@ add_test(function test_processIncoming_n
 
     // There are newly failed records and they are reported.
     do_check_eq(called, 1);
     do_check_eq(counts.failed, 3);
     do_check_eq(counts.applied, 15);
     do_check_eq(counts.newFailed, 3);
     do_check_eq(counts.succeeded, 12);
 
+    // Make sure we recorded telemetry for the failed records.
+    do_check_eq(sumHistogram("WEAVE_ENGINE_APPLY_NEW_FAILURES", { key: "rotary" }), 3);
+
     // Sync again, 1 of the failed items are the same, the rest didn't fail.
     engine._processIncoming();
 
     // Confirming removed failures.
     do_check_attribute_count(engine._store.items, 14);
     do_check_eq(engine.previousFailed.length, 1);
     do_check_eq(engine.previousFailed[0], "record-no-0");
 
     do_check_eq(called, 2);
     do_check_eq(counts.failed, 1);
     do_check_eq(counts.applied, 3);
     do_check_eq(counts.newFailed, 0);
     do_check_eq(counts.succeeded, 2);
 
+    do_check_eq(sumHistogram("WEAVE_ENGINE_APPLY_NEW_FAILURES", { key: "rotary" }), 0);
+
     Svc.Obs.remove("weave:engine:sync:applied", onApplied);
   } finally {
     cleanAndGo(server);
   }
 });
 
 
 add_test(function test_processIncoming_previousFailed() {
@@ -1070,16 +1077,17 @@ add_test(function test_processIncoming_p
     // A second sync with the same failed items should not add the same items again.
     // Items that did not fail a second time should no longer be in previousFailed.
     do_check_attribute_count(engine._store.items, 10);
     do_check_eq(engine.previousFailed.length, 4);
     do_check_eq(engine.previousFailed[0], "record-no-0");
     do_check_eq(engine.previousFailed[1], "record-no-1");
     do_check_eq(engine.previousFailed[2], "record-no-8");
     do_check_eq(engine.previousFailed[3], "record-no-9");
+    do_check_eq(sumHistogram("WEAVE_ENGINE_APPLY_NEW_FAILURES", { key: "rotary" }), 8);
 
     // Refetched items that didn't fail the second time are in engine._store.items.
     do_check_eq(engine._store.items['record-no-4'], "Record No. 4");
     do_check_eq(engine._store.items['record-no-5'], "Record No. 5");
     do_check_eq(engine._store.items['record-no-12'], "Record No. 12");
     do_check_eq(engine._store.items['record-no-13'], "Record No. 13");
   } finally {
     cleanAndGo(server);
@@ -1183,16 +1191,17 @@ add_test(function test_processIncoming_f
     for (let i = 0; i < engine.previousFailed.length; i++) {
       do_check_eq(engine.previousFailed[i], BOGUS_RECORDS[i]);
     }
 
     // Ensure the observer was notified
     do_check_eq(observerData, engine.name);
     do_check_eq(observerSubject.failed, BOGUS_RECORDS.length);
     do_check_eq(observerSubject.newFailed, BOGUS_RECORDS.length);
+    do_check_eq(sumHistogram("WEAVE_ENGINE_APPLY_NEW_FAILURES", { key: "rotary" }), BOGUS_RECORDS.length);
 
     // Testing batching of failed item fetches.
     // Try to sync again. Ensure that we split the request into chunks to avoid
     // URI length limitations.
     function batchDownload(batchSize) {
       count = 0;
       uris  = [];
       engine.guidFetchBatchSize = batchSize;
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -9336,10 +9336,102 @@
   },
   "VIDEO_EME_ADOBE_UNSUPPORTED_REASON": {
     "alert_emails": ["edwin@mozilla.com"],
     "expires_in_version": "50",
     "kind": "enumerated",
     "n_values": 10,
     "releaseChannelCollection": "opt-out",
     "description": "Reason for reporting the Adobe CDM to be unsupported. (1 = NOT_WINDOWS; 2 = WINDOWS_VERSION)"
+  },
+  "FXA_CONFIGURED": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "flag",
+    "releaseChannelCollection": "opt-out",
+    "description": "If the user is signed in to a Firefox Account on this device"
+  },
+  "FXA_UNVERIFIED_ACCOUNT_ERRORS": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "keyed": true,
+    "releaseChannelCollection": "opt-out",
+    "description": "FxA key fetch and certificate signing errors caused by an unverified account. Keyed on the FxA auth server endpoint."
+  },
+  "FXA_HAWK_ERRORS": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "keyed": true,
+    "releaseChannelCollection": "opt-out",
+    "description": "FxA error responses caused by invalid Hawk credentials. Keyed on the FxA auth server endpoint."
+  },
+  "FXA_SERVER_ERRORS": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "keyed": true,
+    "releaseChannelCollection": "opt-out",
+    "description": "400 and 500-class server errors returned by the FxA auth server. Keyed on the endpoint."
+  },
+  "TOKENSERVER_AUTH_ERRORS": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "keyed": true,
+    "releaseChannelCollection": "opt-out",
+    "description": "Token server errors caused by invalid BrowserID assertions. Keyed on the token server error cause."
+  },
+  "WEAVE_ENGINE_APPLY_NEW_FAILURES": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "keyed": true,
+    "releaseChannelCollection": "opt-out",
+    "description": "Number of records that a Sync engine failed to apply. Keyed on the engine name."
+  },
+  "WEAVE_ENGINE_APPLY_FAILURES": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "keyed": true,
+    "releaseChannelCollection": "opt-out",
+    "description": "Failures encountered applying incoming records. Keyed on the engine name."
+  },
+  "WEAVE_ENGINE_SYNC_ERRORS": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "keyed": true,
+    "releaseChannelCollection": "opt-out",
+    "description": "Exceptions thrown by a Sync engine. Keyed on the engine name."
+  },
+  "WEAVE_CAN_FETCH_KEYS": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "flag",
+    "releaseChannelCollection": "opt-out",
+    "description": "Whether Sync keys are present in account storage."
+  },
+  "WEAVE_FXA_KEY_FETCH_ERRORS": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "releaseChannelCollection": "opt-out",
+    "description": "Errors encountered fetching Sync keys, including network errors."
+  },
+  "WEAVE_STORAGE_AUTH_ERRORS": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "keyed": true,
+    "releaseChannelCollection": "opt-out",
+    "description": "Sync storage server authentication errors. Keyed on the Sync record name."
+  },
+  "WEAVE_HMAC_ERRORS": {
+    "alert_emails": ["fx-team@mozilla.com"],
+    "expires_in_version": "45",
+    "kind": "count",
+    "releaseChannelCollection": "opt-out",
+    "description": "Sync cryptoKeys collection HMAC mismatches."
   }
 }