Bug 1454970 - Decouple Remote Settings synchronization from initialization order r=glasserc,mgoodwin
☠☠ backed out by cba6d551e43e ☠ ☠
authorMathieu Leplatre <mathieu@mozilla.com>
Thu, 24 May 2018 23:55:23 +0200
changeset 421304 eadf17764c12fdc70a560a549e649ee4960b0128
parent 421303 a85debb4f781f9f90ba5c9f0df5e27edecf1102d
child 421305 fc040acc00af17b8d8101090739867461f4d843b
push id34091
push userbtara@mozilla.com
push dateTue, 05 Jun 2018 13:52:34 +0000
treeherdermozilla-central@752465b44c79 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglasserc, mgoodwin
bugs1454970
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1454970 - Decouple Remote Settings synchronization from initialization order r=glasserc,mgoodwin MozReview-Commit-ID: LSwFflrFBMn
services/settings/remote-settings.js
services/settings/test/unit/test_remote_settings_poll.js
--- a/services/settings/remote-settings.js
+++ b/services/settings/remote-settings.js
@@ -7,17 +7,17 @@
 var EXPORTED_SYMBOLS = [
   "RemoteSettings",
   "jexlFilterFunc"
 ];
 
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm", {});
-Cu.importGlobalProperties(["fetch"]);
+Cu.importGlobalProperties(["fetch", "indexedDB"]);
 
 ChromeUtils.defineModuleGetter(this, "Kinto",
                                "resource://services-common/kinto-offline-client.js");
 ChromeUtils.defineModuleGetter(this, "KintoHttpClient",
                                "resource://services-common/kinto-http-client.js");
 ChromeUtils.defineModuleGetter(this, "CanonicalJSON",
                                "resource://gre/modules/CanonicalJSON.jsm");
 ChromeUtils.defineModuleGetter(this, "UptakeTelemetry",
@@ -173,16 +173,31 @@ async function fetchLatestChanges(url, l
     if (!isNaN(value)) {
       backoffSeconds = value;
     }
   }
 
   return {changes, currentEtag, serverTimeMillis, backoffSeconds};
 }
 
+/**
+ * Load the the JSON file distributed with the release for this collection.
+ */
+async function loadDumpFile(filename) {
+  // Replace OS specific path separator by / for URI.
+  const { components: folderFile } = OS.Path.split(filename);
+  const fileURI = `resource://app/defaults/settings/${folderFile.join("/")}`;
+  const response = await fetch(fileURI);
+  if (!response.ok) {
+    throw new Error(`Could not read from '${fileURI}'`);
+  }
+  // Will be rejected if JSON is invalid.
+  return response.json();
+}
+
 
 class RemoteSettingsClient {
 
   constructor(collectionName, { bucketName, signerName, filterFunc = jexlFilterFunc, lastCheckTimePref }) {
     this.collectionName = collectionName;
     this.bucketName = bucketName;
     this.signerName = signerName;
     this.filterFunc = filterFunc;
@@ -270,18 +285,18 @@ class RemoteSettingsClient {
     const { filters = {}, order } = options;
     const c = await this.openCollection();
 
     const timestamp = await c.db.getLastModified();
     // If the local database was never synchronized, then we attempt to load
     // a packaged JSON dump.
     if (timestamp == null) {
       try {
-         const { data } = await this._loadDumpFile();
-         await c.loadDump(data);
+        const { data } = await loadDumpFile(this.filename);
+        await c.loadDump(data);
       } catch (e) {
         // Report but return an empty list since there will be no data anyway.
         Cu.reportError(e);
         return [];
       }
     }
 
     const { data } = await c.list({ filters, order });
@@ -321,17 +336,17 @@ class RemoteSettingsClient {
       let collectionLastModified = await collection.db.getLastModified();
 
       // If there is no data currently in the collection, attempt to import
       // initial data from the application defaults.
       // This allows to avoid synchronizing the whole collection content on
       // cold start.
       if (!collectionLastModified && loadDump) {
         try {
-          const initialData = await this._loadDumpFile();
+          const initialData = await loadDumpFile(this.filename);
           await collection.loadDump(initialData.data);
           collectionLastModified = await collection.db.getLastModified();
         } catch (e) {
           // Report but go-on.
           Cu.reportError(e);
         }
       }
 
@@ -457,31 +472,16 @@ class RemoteSettingsClient {
       if (reportStatus === null) {
         reportStatus = UptakeTelemetry.STATUS.SUCCESS;
       }
       // Report success/error status to Telemetry.
       UptakeTelemetry.report(this.identifier, reportStatus);
     }
   }
 
-  /**
-   * Load the the JSON file distributed with the release for this collection.
-   */
-  async _loadDumpFile() {
-    // Replace OS specific path separator by / for URI.
-    const { components: folderFile } = OS.Path.split(this.filename);
-    const fileURI = `resource://app/defaults/settings/${folderFile.join("/")}`;
-    const response = await fetch(fileURI);
-    if (!response.ok) {
-      throw new Error(`Could not read from '${fileURI}'`);
-    }
-    // Will be rejected if JSON is invalid.
-    return response.json();
-  }
-
   async _validateCollectionSignature(remote, payload, collection, options = {}) {
     const {ignoreLocal} = options;
     // this is a content-signature field from an autograph response.
     const signaturePayload = await fetchCollectionMetadata(remote, collection);
     if (!signaturePayload) {
       throw new Error(MISSING_SIGNATURE);
     }
     const {x5u, signature} = signaturePayload;
@@ -534,16 +534,60 @@ class RemoteSettingsClient {
     }
     const environment = cacheProxy(ClientEnvironment);
     const dataPromises = data.map(e => this.filterFunc(e, environment));
     const results = await Promise.all(dataPromises);
     return results.filter(v => !!v);
   }
 }
 
+/**
+ * Check if an IndexedDB database exists for the specified bucket and collection.
+ *
+ * @param {String} bucket
+ * @param {String} collection
+ * @return {bool} Whether it exists or not.
+ */
+async function databaseExists(bucket, collection) {
+  // The dbname is chosen by kinto.js from the bucket and collection names.
+  // https://github.com/Kinto/kinto.js/blob/41aa1526e/src/collection.js#L231
+  const dbname = `${bucket}/${collection}`;
+  try {
+    await new Promise((resolve, reject) => {
+      const request = indexedDB.open(dbname, 1);
+      request.onupgradeneeded = event => {
+        event.target.transaction.abort();
+        reject(event.target.error);
+      };
+      request.onerror = event => reject(event.target.error);
+      request.onsuccess = event => resolve(event.target.result);
+    });
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+/**
+ * Check if we ship a JSON dump for the specified bucket and collection.
+ *
+ * @param {String} bucket
+ * @param {String} collection
+ * @return {bool} Whether it is present or not.
+ */
+async function hasLocalDump(bucket, collection) {
+  const filename = OS.Path.join(bucket, `${collection}.json`);
+  try {
+    await loadDumpFile(filename);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
 
 function remoteSettingsFunction() {
   const _clients = new Map();
 
   // If not explicitly specified, use the default bucket name and signer.
   const mainBucket = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_BUCKET);
   const defaultSigner = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_SIGNER);
 
@@ -627,30 +671,54 @@ function remoteSettingsFunction() {
     Services.prefs.setIntPref(PREF_SETTINGS_CLOCK_SKEW_SECONDS, clockDifference);
     Services.prefs.setIntPref(PREF_SETTINGS_LAST_UPDATE, serverTimeMillis / 1000);
 
     const loadDump = Services.prefs.getBoolPref(PREF_SETTINGS_LOAD_DUMP, true);
     // Iterate through the collections version info and initiate a synchronization
     // on the related remote settings client.
     let firstError;
     for (const change of changes) {
-      const {bucket, collection, last_modified: lastModified} = change;
-      const key = `${bucket}/${collection}`;
-      if (!_clients.has(key)) {
+      const {bucket: bucketName, collection, last_modified: lastModified} = change;
+      const key = `${bucketName}/${collection}`;
+
+      let client;
+      // Check if a client was registered for this bucket/collection. Potentially
+      // with some specific options like bucket, signer, etc.
+      if (_clients.has(key)) {
+        client = _clients.get(key);
+        // If the bucket name was changed manually on the client instance and does not
+        // match, it should be skipped.
+        if (client.bucketName != bucketName) {
+          continue;
+        }
+
+      // There was no client registered for this bucket/collection, but it's the main bucket,
+      // therefore we can instantiate a client with the default options.
+      // So if we have a local database or if we ship a JSON dump, then it means that
+      // this client is known but it was not registered yet (eg. calling module not "imported" yet).
+      } else if (bucketName == mainBucket && (await databaseExists(bucketName, collection) ||
+                                              await hasLocalDump(bucketName, collection))) {
+        client = new RemoteSettingsClient(collection, {bucketName, signerName: defaultSigner});
+
+      // We are not able to synchronize data for clients in specific buckets since we cannot
+      // guess which `signerName` has to be used for example.
+      // And we don't want to synchronize data for collections in the main bucket that are
+      // completely unknown (ie. no database and no JSON dump).
+      } else {
         continue;
       }
-      const client = _clients.get(key);
-      if (client.bucketName != bucket) {
-        continue;
-      }
+
+      // Start synchronization! It will be a no-op if the specified `lastModified` equals
+      // the one in the local database.
       try {
         await client.maybeSync(lastModified, serverTimeMillis, {loadDump});
       } catch (e) {
         if (!firstError) {
           firstError = e;
+          firstError.details = change;
         }
       }
     }
     if (firstError) {
       // cause the promise to reject by throwing the first observed error
       throw firstError;
     }
 
--- a/services/settings/test/unit/test_remote_settings_poll.js
+++ b/services/settings/test/unit/test_remote_settings_poll.js
@@ -1,239 +1,396 @@
 /* import-globals-from ../../../common/tests/unit/head_helpers.js */
 
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://testing-common/httpd.js");
 
 const { UptakeTelemetry } = ChromeUtils.import("resource://services-common/uptake-telemetry.js", {});
 const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js", {});
+const { Kinto } = ChromeUtils.import("resource://services-common/kinto-offline-client.js", {});
 
 var server;
 
 const PREF_SETTINGS_SERVER = "services.settings.server";
 const PREF_SETTINGS_SERVER_BACKOFF = "services.settings.server.backoff";
 const PREF_LAST_UPDATE = "services.settings.last_update_seconds";
 const PREF_LAST_ETAG = "services.settings.last_etag";
 const PREF_CLOCK_SKEW_SECONDS = "services.settings.clock_skew_seconds";
 
 // Telemetry report result.
 const TELEMETRY_HISTOGRAM_KEY = "settings-changes-monitoring";
-
-// Check to ensure maybeSync is called with correct values when a changes
-// document contains information on when a collection was last modified
-add_task(async function test_check_maybeSync() {
-  const changesPath = "/v1/buckets/monitor/collections/changes/records";
-
-  // register a handler
-  function handleResponse(serverTimeMillis, request, response) {
-    try {
-      const sampled = getSampleResponse(request, server.identity.primaryPort);
-      if (!sampled) {
-        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
-      }
+const CHANGES_PATH = "/v1/buckets/monitor/collections/changes/records";
 
-      response.setStatusLine(null, sampled.status.status,
-                             sampled.status.statusText);
-      // send the headers
-      for (let headerLine of sampled.sampleHeaders) {
-        let headerElements = headerLine.split(":");
-        response.setHeader(headerElements[0], headerElements[1].trimLeft());
-      }
 
-      // set the server date
-      response.setHeader("Date", (new Date(serverTimeMillis)).toUTCString());
-
-      response.write(sampled.responseBody);
-    } catch (e) {
-      dump(`${e}\n`);
-    }
-  }
-
-  server.registerPathHandler(changesPath, handleResponse.bind(null, 2000));
-
+async function clear_state() {
   // set up prefs so the kinto updater talks to the test server
   Services.prefs.setCharPref(PREF_SETTINGS_SERVER,
     `http://localhost:${server.identity.primaryPort}/v1`);
 
   // set some initial values so we can check these are updated appropriately
   Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
   Services.prefs.setIntPref(PREF_CLOCK_SKEW_SECONDS, 0);
   Services.prefs.clearUserPref(PREF_LAST_ETAG);
-
-
-  let startTime = Date.now();
-
-  // ensure we get the maybeSync call
-  // add a test kinto client that will respond to lastModified information
-  // for a collection called 'test-collection'
-  const c = RemoteSettings("test-collection", {
-    bucketName: "test-bucket",
-  });
-  c.maybeSync = () => {};
-
-  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
-
-  let notificationObserved = false;
-
-  // Ensure that the remote-settings-changes-polled notification works
-  let certblockObserver = {
-    observe(aSubject, aTopic, aData) {
-      Services.obs.removeObserver(this, "remote-settings-changes-polled");
-      notificationObserved = true;
-    }
-  };
-
-  Services.obs.addObserver(certblockObserver, "remote-settings-changes-polled");
-
-  await RemoteSettings.pollChanges();
-
-  Assert.ok(notificationObserved, "a notification should have been observed");
-
-  // check the last_update is updated
-  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
-
-  // How does the clock difference look?
-  let endTime = Date.now();
-  let clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
-  // we previously set the serverTime to 2 (seconds past epoch)
-  Assert.ok(clockDifference <= endTime / 1000
-              && clockDifference >= Math.floor(startTime / 1000) - 2);
-  // Last timestamp was saved. An ETag header value is a quoted string.
-  let lastEtag = Services.prefs.getCharPref(PREF_LAST_ETAG);
-  Assert.equal(lastEtag, "\"1100\"");
-
-  // Simulate a poll with up-to-date collection.
-  Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
-  // If server has no change, a 304 is received, maybeSync() is not called.
-  c.maybeSync = () => { throw new Error("Should not be called"); };
-  await RemoteSettings.pollChanges();
-  // Last update is overwritten
-  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
-
-
-  // Simulate a server error.
-  function simulateErrorResponse(request, response) {
-    response.setHeader("Date", (new Date(3000)).toUTCString());
-    response.setHeader("Content-Type", "application/json; charset=UTF-8");
-    response.write(JSON.stringify({
-      code: 503,
-      errno: 999,
-      error: "Service Unavailable",
-    }));
-    response.setStatusLine(null, 503, "Service Unavailable");
-  }
-  server.registerPathHandler(changesPath, simulateErrorResponse);
+}
 
-  // pollChanges() fails with adequate error and no notification.
-  let error;
-  notificationObserved = false;
-  Services.obs.addObserver(certblockObserver, "remote-settings-changes-polled");
-  try {
-    await RemoteSettings.pollChanges();
-  } catch (e) {
-    error = e;
-  }
-  Assert.ok(!notificationObserved, "a notification should not have been observed");
-  Assert.ok(/Polling for changes failed/.test(error.message));
-  // When an error occurs, last update was not overwritten (see Date header above).
-  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
-
-  // check negative clock skew times
-
-  // set to a time in the future
-  server.registerPathHandler(changesPath, handleResponse.bind(null, Date.now() + 10000));
-
-  await RemoteSettings.pollChanges();
-
-  clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
-  // we previously set the serverTime to Date.now() + 10000 ms past epoch
-  Assert.ok(clockDifference <= 0 && clockDifference >= -10);
-
-  //
-  // Backoff
-  //
-  function simulateBackoffResponse(request, response) {
-    response.setHeader("Content-Type", "application/json; charset=UTF-8");
-    response.setHeader("Backoff", "10");
-    response.write(JSON.stringify({data: []}));
+function serveChangesEntries(serverTime, entries) {
+  return (request, response) => {
     response.setStatusLine(null, 200, "OK");
-  }
-  server.registerPathHandler(changesPath, simulateBackoffResponse);
-  // First will work.
-  await RemoteSettings.pollChanges();
-  // Second will fail because we haven't waited.
-  try {
-    await RemoteSettings.pollChanges();
-    // The previous line should have thrown an error.
-    Assert.ok(false);
-  } catch (e) {
-    Assert.ok(/Server is asking clients to back off; retry in \d+s./.test(e.message));
-  }
-  // Once backoff time has expired, polling for changes can start again.
-  server.registerPathHandler(changesPath, handleResponse.bind(null, 2000));
-  Services.prefs.setCharPref(PREF_SETTINGS_SERVER_BACKOFF, `${Date.now() - 1000}`);
-  await RemoteSettings.pollChanges();
-  // Backoff tracking preference was cleared.
-  Assert.ok(!Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF));
-
-
-  // Simulate a network error (to check telemetry report).
-  Services.prefs.setCharPref(PREF_SETTINGS_SERVER, "http://localhost:42/v1");
-  try {
-    await RemoteSettings.pollChanges();
-  } catch (e) {}
-
-  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
-  // ensure that we've accumulated the correct telemetry
-  const expectedIncrements = {
-    [UptakeTelemetry.STATUS.UP_TO_DATE]: 4,
-    [UptakeTelemetry.STATUS.SUCCESS]: 1,
-    [UptakeTelemetry.STATUS.BACKOFF]: 1,
-    [UptakeTelemetry.STATUS.SERVER_ERROR]: 1,
-    [UptakeTelemetry.STATUS.NETWORK_ERROR]: 1,
-    [UptakeTelemetry.STATUS.UNKNOWN_ERROR]: 0,
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setHeader("Date", (new Date(serverTime)).toUTCString());
+    if (entries.length) {
+      response.setHeader("ETag", `"${entries[0].last_modified}"`);
+    }
+    response.write(JSON.stringify({"data": entries}));
   };
-  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
-});
+}
 
 function run_test() {
   // Set up an HTTP Server
   server = new HttpServer();
   server.start(-1);
 
   run_next_test();
 
   registerCleanupFunction(function() {
     server.stop(function() { });
   });
 }
 
-// get a response for a given request from sample data
-function getSampleResponse(req, port) {
-  const responses = {
-    "GET:/v1/buckets/monitor/collections/changes/records?": {
-      "sampleHeaders": [
-        "Content-Type: application/json; charset=UTF-8",
-        "ETag: \"1100\""
-      ],
-      "status": {status: 200, statusText: "OK"},
-      "responseBody": JSON.stringify({"data": [{
-        "host": "localhost",
-        "last_modified": 1100,
-        "bucket": "test-bucket-aurora",
-        "id": "330a0c5f-fadf-ff0b-40c8-4eb0d924ff6a",
-        "collection": "test-collection"
-      }, {
-        "host": "localhost",
-        "last_modified": 1000,
-        "bucket": "test-bucket",
-        "id": "254cbb9e-6888-4d9f-8e60-58b74faa8778",
-        "collection": "test-collection"
-      }]})
+add_task(clear_state);
+
+add_task(async function test_check_success() {
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const serverTime = 8000;
+
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(serverTime, [{
+    id: "330a0c5f-fadf-ff0b-40c8-4eb0d924ff6a",
+    last_modified: 1100,
+    host: "localhost",
+    bucket: "some-other-bucket",
+    collection: "test-collection"
+  }, {
+    id: "254cbb9e-6888-4d9f-8e60-58b74faa8778",
+    last_modified: 1000,
+    host: "localhost",
+    bucket: "test-bucket",
+    collection: "test-collection"
+  }]));
+
+  // add a test kinto client that will respond to lastModified information
+  // for a collection called 'test-collection'
+  let maybeSyncCalled = false;
+  const c = RemoteSettings("test-collection", {
+    bucketName: "test-bucket",
+  });
+  c.maybeSync = () => { maybeSyncCalled = true; };
+
+  // Ensure that the remote-settings-changes-polled notification works
+  let notificationObserved = false;
+  const observer = {
+    observe(aSubject, aTopic, aData) {
+      Services.obs.removeObserver(this, "remote-settings-changes-polled");
+      notificationObserved = true;
+    }
+  };
+  Services.obs.addObserver(observer, "remote-settings-changes-polled");
+
+  await RemoteSettings.pollChanges();
+
+  // It didn't fail, hence we are sure that the unknown collection ``some-other-bucket/test-collection``
+  // was ignored, otherwise it would have tried to reach the network.
+
+  Assert.ok(maybeSyncCalled, "maybeSync was called");
+  Assert.ok(notificationObserved, "a notification should have been observed");
+  // Last timestamp was saved. An ETag header value is a quoted string.
+  Assert.equal(Services.prefs.getCharPref(PREF_LAST_ETAG), "\"1100\"");
+  // check the last_update is updated
+  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), serverTime / 1000);
+  // ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.SUCCESS]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_check_up_to_date() {
+  // Simulate a poll with up-to-date collection.
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+
+  const serverTime = 4000;
+  function server304(request, response) {
+    if (request.hasHeader("if-none-match") && request.getHeader("if-none-match") == "\"1100\"") {
+      response.setHeader("Date", (new Date(serverTime)).toUTCString());
+      response.setStatusLine(null, 304, "Service Not Modified");
+    }
+  }
+  server.registerPathHandler(CHANGES_PATH, server304);
+
+  Services.prefs.setCharPref(PREF_LAST_ETAG, '"1100"');
+
+  // Ensure that the remote-settings-changes-polled notification is sent.
+  let notificationObserved = false;
+  const observer = {
+    observe(aSubject, aTopic, aData) {
+      Services.obs.removeObserver(this, "remote-settings-changes-polled");
+      notificationObserved = true;
+    }
+  };
+  Services.obs.addObserver(observer, "remote-settings-changes-polled");
+
+  // If server has no change, a 304 is received, maybeSync() is not called.
+  let maybeSyncCalled = false;
+  const c = RemoteSettings("test-collection", {
+    bucketName: "test-bucket",
+  });
+  c.maybeSync = () => { maybeSyncCalled = true; };
+
+  await RemoteSettings.pollChanges();
+
+  Assert.ok(notificationObserved, "a notification should have been observed");
+  Assert.ok(!maybeSyncCalled, "maybeSync should not be called");
+  // Last update is overwritten
+  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), serverTime / 1000);
+
+  // ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.UP_TO_DATE]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_server_error() {
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+
+  // Simulate a server error.
+  function simulateErrorResponse(request, response) {
+    response.setHeader("Date", (new Date(3000)).toUTCString());
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.write(JSON.stringify({
+      code: 503,
+      errno: 999,
+      error: "Service Unavailable",
+    }));
+    response.setStatusLine(null, 503, "Service Unavailable");
+  }
+  server.registerPathHandler(CHANGES_PATH, simulateErrorResponse);
+
+  let notificationObserved = false;
+  const observer = {
+    observe(aSubject, aTopic, aData) {
+      Services.obs.removeObserver(this, "remote-settings-changes-polled");
+      notificationObserved = true;
     }
   };
+  Services.obs.addObserver(observer, "remote-settings-changes-polled");
+  Services.prefs.setIntPref(PREF_LAST_UPDATE, 42);
 
-  if (req.hasHeader("if-none-match") && req.getHeader("if-none-match", "") == "\"1100\"")
-    return {sampleHeaders: [], status: {status: 304, statusText: "Not Modified"}, responseBody: ""};
+  // pollChanges() fails with adequate error and no notification.
+  let error;
+  try {
+    await RemoteSettings.pollChanges();
+  } catch (e) {
+    error = e;
+  }
+
+  Assert.ok(!notificationObserved, "a notification should not have been observed");
+  Assert.ok(/Polling for changes failed/.test(error.message));
+  // When an error occurs, last update was not overwritten.
+  Assert.equal(Services.prefs.getIntPref(PREF_LAST_UPDATE), 42);
+  // ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.SERVER_ERROR]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_check_clockskew_is_updated() {
+  const serverTime = 2000;
+
+  function serverResponse(request, response) {
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setHeader("Date", (new Date(serverTime)).toUTCString());
+    response.write(JSON.stringify({data: []}));
+    response.setStatusLine(null, 200, "OK");
+  }
+  server.registerPathHandler(CHANGES_PATH, serverResponse);
+
+  let startTime = Date.now();
+
+  await RemoteSettings.pollChanges();
+
+  // How does the clock difference look?
+  let endTime = Date.now();
+  let clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
+  // we previously set the serverTime to 2 (seconds past epoch)
+  Assert.ok(clockDifference <= endTime / 1000
+              && clockDifference >= Math.floor(startTime / 1000) - (serverTime / 1000));
+
+  // check negative clock skew times
+  // set to a time in the future
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(Date.now() + 10000, []));
+
+  await RemoteSettings.pollChanges();
+
+  clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
+  // we previously set the serverTime to Date.now() + 10000 ms past epoch
+  Assert.ok(clockDifference <= 0 && clockDifference >= -10);
+});
+add_task(clear_state);
+
+
+add_task(async function test_backoff() {
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+
+  function simulateBackoffResponse(request, response) {
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setHeader("Backoff", "10");
+    response.write(JSON.stringify({data: []}));
+    response.setStatusLine(null, 200, "OK");
+  }
+  server.registerPathHandler(CHANGES_PATH, simulateBackoffResponse);
+
+  // First will work.
+  await RemoteSettings.pollChanges();
+  // Second will fail because we haven't waited.
+  try {
+    await RemoteSettings.pollChanges();
+    // The previous line should have thrown an error.
+    Assert.ok(false);
+  } catch (e) {
+    Assert.ok(/Server is asking clients to back off; retry in \d+s./.test(e.message));
+  }
+
+  // Once backoff time has expired, polling for changes can start again.
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(12000, [{
+    id: "6a733d4a-601e-11e8-837a-0f85257529a1",
+    last_modified: 1300,
+    host: "localhost",
+    bucket: "some-bucket",
+    collection: "some-collection"
+  }]));
+  Services.prefs.setCharPref(PREF_SETTINGS_SERVER_BACKOFF, `${Date.now() - 1000}`);
+
+  await RemoteSettings.pollChanges();
+
+  // Backoff tracking preference was cleared.
+  Assert.ok(!Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF));
 
-  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
-         responses[req.method];
-}
+  // Ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.SUCCESS]: 1,
+    [UptakeTelemetry.STATUS.UP_TO_DATE]: 1,
+    [UptakeTelemetry.STATUS.BACKOFF]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_network_error() {
+  const startHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+
+  // Simulate a network error (to check telemetry report).
+  Services.prefs.setCharPref(PREF_SETTINGS_SERVER, "http://localhost:42/v1");
+  try {
+    await RemoteSettings.pollChanges();
+  } catch (e) {}
+
+  // ensure that we've accumulated the correct telemetry
+  const endHistogram = getUptakeTelemetrySnapshot(TELEMETRY_HISTOGRAM_KEY);
+  const expectedIncrements = {
+    [UptakeTelemetry.STATUS.NETWORK_ERROR]: 1,
+  };
+  checkUptakeTelemetry(startHistogram, endHistogram, expectedIncrements);
+});
+add_task(clear_state);
+
+
+add_task(async function test_syncs_clients_with_local_database() {
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(42000, [{
+    id: "d4a14f44-601f-11e8-8b8a-030f3dc5b844",
+    last_modified: 10000,
+    host: "localhost",
+    bucket: "main",
+    collection: "some-unknown"
+  }, {
+    id: "39f57e4e-6023-11e8-8b74-77c8dedfb389",
+    last_modified: 9000,
+    host: "localhost",
+    bucket: "blocklists",
+    collection: "addons"
+  }, {
+    id: "9a594c1a-601f-11e8-9c8a-33b2239d9113",
+    last_modified: 8000,
+    host: "localhost",
+    bucket: "main",
+    collection: "recipes"
+  }]));
+
+  // This simulates what remote-settings would do when initializing a local database.
+  // We don't want to instantiate a client using the RemoteSettings() API
+  // since we want to test «unknown» clients that have a local database.
+  await (new Kinto.adapters.IDB("blocklists/addons")).saveLastModified(42);
+  await (new Kinto.adapters.IDB("main/recipes")).saveLastModified(43);
+
+  let error;
+  try {
+    await RemoteSettings.pollChanges();
+  } catch (e) {
+    error = e;
+  }
+
+  // The `main/some-unknown` should be skipped because it has no local database.
+  // The `blocklists/addons` should be skipped because it is not the main bucket.
+  // The `recipes` has a local database, and should cause a network error because the test
+  // does not setup the server to receive the requests of `maybeSync()`.
+  Assert.ok(/HTTP 404/.test(error.message), "server will return 404 on sync");
+  Assert.equal(error.details.collection, "recipes");
+});
+add_task(clear_state);
+
+
+add_task(async function test_syncs_clients_with_local_dump() {
+  server.registerPathHandler(CHANGES_PATH, serveChangesEntries(42000, [{
+    id: "d4a14f44-601f-11e8-8b8a-030f3dc5b844",
+    last_modified: 10000,
+    host: "localhost",
+    bucket: "main",
+    collection: "some-unknown"
+  }, {
+    id: "39f57e4e-6023-11e8-8b74-77c8dedfb389",
+    last_modified: 9000,
+    host: "localhost",
+    bucket: "blocklists",
+    collection: "addons"
+  }, {
+    id: "9a594c1a-601f-11e8-9c8a-33b2239d9113",
+    last_modified: 8000,
+    host: "localhost",
+    bucket: "main",
+    collection: "tippytop"
+  }]));
+
+  let error;
+  try {
+    await RemoteSettings.pollChanges();
+  } catch (e) {
+    error = e;
+  }
+
+  // The `main/some-unknown` should be skipped because it has no dump.
+  // The `blocklists/addons` should be skipped because it is not the main bucket.
+  // The `tippytop` has a dump, and should cause a network error because the test
+  // does not setup the server to receive the requests of `maybeSync()`.
+  Assert.ok(/HTTP 404/.test(error.message), "server will return 404 on sync");
+  Assert.equal(error.details.collection, "tippytop");
+});
+add_task(clear_state);