Backed out changeset 81abb539b39c (bug 1502146) for failing services/settings/test/unit/test_remote_settings_worker.js on a CLOSED TREE
authorAndreea Pavel <apavel@mozilla.com>
Tue, 20 Nov 2018 02:13:24 +0200
changeset 447085 2b2fd36ea6c1ab124817d6826af6d38d3f1306b7
parent 447084 104d7f34de53a9359d6a1983b8d421a60a649592
child 447086 817c31467dcb1ee977a92cf9e07af04da8dbe343
push id73322
push userapavel@mozilla.com
push dateTue, 20 Nov 2018 00:13:54 +0000
treeherderautoland@2b2fd36ea6c1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1502146
milestone65.0a1
backs out81abb539b39c8d2f483d18234bd6014d6e521fc6
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 81abb539b39c (bug 1502146) for failing services/settings/test/unit/test_remote_settings_worker.js on a CLOSED TREE
services/common/kinto-offline-client.js
services/settings/RemoteSettingsWorker.js
services/settings/RemoteSettingsWorker.jsm
services/settings/moz.build
services/settings/remote-settings.js
services/settings/test/unit/test_remote_settings_worker.js
services/settings/test/unit/xpcshell.ini
toolkit/modules/CanonicalJSON.jsm
tools/lint/eslint/modules.json
--- a/services/common/kinto-offline-client.js
+++ b/services/common/kinto-offline-client.js
@@ -28,17 +28,17 @@
 //
 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1394556#c3 for
 // more details.
 const global = this;
 
 var EXPORTED_SYMBOLS = ["Kinto"];
 
 /*
- * Version 12.2.4 - 8fb687a
+ * Version 12.2.0 - 266e100
  */
 
 (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Kinto = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(require,module,exports){
 /*
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -66,25 +66,26 @@ var _IDB = _interopRequireDefault(requir
 
 var _utils = require("../src/utils");
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 ChromeUtils.import("resource://gre/modules/Timer.jsm");
 ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 XPCOMUtils.defineLazyGlobalGetters(global, ["fetch", "indexedDB"]);
-ChromeUtils.defineModuleGetter(global, "EventEmitter", "resource://gre/modules/EventEmitter.jsm"); // Use standalone kinto-http module landed in FFx.
-
-ChromeUtils.defineModuleGetter(global, "KintoHttpClient", "resource://services-common/kinto-http-client.js");
-XPCOMUtils.defineLazyGetter(global, "generateUUID", () => {
-  const {
-    generateUUID
-  } = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator);
-  return generateUUID;
-});
+const {
+  EventEmitter
+} = ChromeUtils.import("resource://gre/modules/EventEmitter.jsm", {});
+const {
+  generateUUID
+} = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator); // Use standalone kinto-http module landed in FFx.
+
+const {
+  KintoHttpClient
+} = ChromeUtils.import("resource://services-common/kinto-http-client.js");
 
 class Kinto extends _KintoBase.default {
   static get adapters() {
     return {
       BaseAdapter: _base.default,
       IDB: _IDB.default
     };
   }
@@ -484,59 +485,55 @@ const cursorHandlers = {
  * @return {IDBRequest}
  */
 
 function createListRequest(cid, store, filters, done) {
   const filterFields = Object.keys(filters); // If no filters, get all results in one bulk.
 
   if (filterFields.length == 0) {
     const request = store.index("cid").getAll(IDBKeyRange.only(cid));
-
     request.onsuccess = event => done(event.target.result);
-
     return request;
-  } // Introspect filters and check if they leverage an indexed field.
-
-
+  }
+  // Introspect filters and check if they leverage an indexed field.
   const indexField = filterFields.find(field => {
     return INDEXED_FIELDS.includes(field);
   });
 
   if (!indexField) {
     // Iterate on all records for this collection (ie. cid)
     const request = store.index("cid").openCursor(IDBKeyRange.only(cid));
     request.onsuccess = cursorHandlers.all(filters, done);
     return request;
   } // If `indexField` was used already, don't filter again.
 
 
-  const remainingFilters = (0, _utils.omitKeys)(filters, [indexField]); // value specified in the filter (eg. `filters: { _status: ["created", "updated"] }`)
-
-  const value = filters[indexField]; // For the "id" field, use the primary key.
-
-  const indexStore = indexField == "id" ? store : store.index(indexField); // WHERE IN equivalent clause
-
+  const remainingFilters = (0, _utils.omitKeys)(filters, indexField); // value specified in the filter (eg. `filters: { _status: ["created", "updated"] }`)
+
+  const value = filters[indexField];
+  // For the "id" field, use the primary key.
+  const indexStore = indexField == "id" ? store : store.index(indexField);
+
+  // WHERE IN equivalent clause
   if (Array.isArray(value)) {
     if (value.length === 0) {
       return done([]);
     }
 
     const values = value.map(i => [cid, i]).sort();
     const range = IDBKeyRange.bound(values[0], values[values.length - 1]);
     const request = indexStore.openCursor(range);
     request.onsuccess = cursorHandlers.in(values, remainingFilters, done);
     return request;
-  } // If no filters on custom attribute, get all results in one bulk.
-
-
+  }
+
+  // If no filters on custom attribute, get all results in one bulk.
   if (remainingFilters.length == 0) {
     const request = indexStore.getAll(IDBKeyRange.only([cid, value]));
-
     request.onsuccess = event => done(event.target.result);
-
     return request;
   } // WHERE field = value clause
 
 
   const request = indexStore.openCursor(IDBKeyRange.only([cid, value]));
   request.onsuccess = cursorHandlers.all(remainingFilters, done);
   return request;
 }
@@ -765,23 +762,20 @@ class IDB extends _base.default {
       } // Preload specified records using a list request.
 
 
       const filters = {
         id: options.preload
       };
       createListRequest(this.cid, store, filters, records => {
         // Store obtained records by id.
-        const preloaded = {};
-
-        for (const record of records) {
-          delete record["_cid"];
-          preloaded[record.id] = record;
-        }
-
+        const preloaded = records.reduce((acc, record) => {
+          acc[record.id] = (0, _utils.omitKeys)(record, ["_cid"]);
+          return acc;
+        }, {});
         runCallback(preloaded);
       });
     }, {
       mode: "readwrite"
     });
     return result;
   }
   /**
@@ -821,21 +815,17 @@ class IDB extends _base.default {
     } = params;
 
     try {
       let results = [];
       await this.prepare("records", store => {
         createListRequest(this.cid, store, filters, _results => {
           // we have received all requested records that match the filters,
           // we now park them within current scope and hide the `_cid` attribute.
-          for (const result of _results) {
-            delete result["_cid"];
-          }
-
-          results = _results;
+          results = _results.map(r => (0, _utils.omitKeys)(r, ["_cid"]));
         });
       }); // The resulting list of records is sorted.
       // XXX: with some efforts, this could be fully implemented using IDB API.
 
       return params.order ? (0, _utils.sortObjects)(params.order, results) : results;
     } catch (e) {
       this._handleError("list", e);
     }
@@ -890,31 +880,17 @@ class IDB extends _base.default {
    * @param  {Array} records The records to load.
    * @return {Promise}
    */
 
 
   async loadDump(records) {
     try {
       await this.execute(transaction => {
-        // Since the put operations are asynchronous, we chain
-        // them together. The last one will be waited for the
-        // `transaction.oncomplete` callback. (see #execute())
-        let i = 0;
-        putNext();
-
-        function putNext() {
-          if (i == records.length) {
-            return;
-          } // On error, `transaction.onerror` is called.
-
-
-          transaction.update(records[i]).onsuccess = putNext;
-          ++i;
-        }
+        records.forEach(record => transaction.update(record));
       });
       const previousLastModified = await this.getLastModified();
       const lastModified = Math.max(...records.map(record => record.last_modified));
 
       if (lastModified > previousLastModified) {
         await this.saveLastModified(lastModified);
       }
 
@@ -943,17 +919,17 @@ function transactionProxy(adapter, store
   return {
     create(record) {
       store.add({ ...record,
         _cid
       });
     },
 
     update(record) {
-      return store.put({ ...record,
+      store.put({ ...record,
         _cid
       });
     },
 
     delete(id) {
       store.delete([_cid, id]);
     },
 
@@ -3135,24 +3111,23 @@ function deepEqual(a, b) {
  *
  * @param  {Object} obj        The original object.
  * @param  {Array}  keys       The list of keys to exclude.
  * @return {Object}            A copy without the specified keys.
  */
 
 
 function omitKeys(obj, keys = []) {
-  const result = { ...obj
-  };
-
-  for (const key of keys) {
-    delete result[key];
-  }
-
-  return result;
+  return Object.keys(obj).reduce((acc, key) => {
+    if (!keys.includes(key)) {
+      acc[key] = obj[key];
+    }
+
+    return acc;
+  }, {});
 }
 
 function arrayEqual(a, b) {
   if (a.length !== b.length) {
     return false;
   }
 
   for (let i = a.length; i--;) {
deleted file mode 100644
--- a/services/settings/RemoteSettingsWorker.js
+++ /dev/null
@@ -1,188 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-/* eslint-env mozilla/chrome-worker */
-
-"use strict";
-
-/**
- * A worker dedicated to Remote Settings.
- */
-
-importScripts("resource://gre/modules/workers/require.js",
-              "resource://gre/modules/CanonicalJSON.jsm",
-              "resource://gre/modules/third_party/jsesc/jsesc.js");
-
-const IDB_NAME = "remote-settings";
-const IDB_VERSION = 1;
-const IDB_RECORDS_STORE = "records";
-const IDB_TIMESTAMPS_STORE = "timestamps";
-
-const Agent = {
-  /**
-   * Return the canonical JSON serialization of the changes
-   * applied to the local records.
-   * It has to match what is done on the server (See Kinto/kinto-signer).
-   *
-   * @param {Array<Object>} localRecords
-   * @param {Array<Object>} remoteRecords
-   * @param {String} timestamp
-   * @returns {String}
-   */
-  async canonicalStringify(localRecords, remoteRecords, timestamp) {
-    // Sort list by record id.
-    let allRecords = localRecords.concat(remoteRecords).sort((a, b) => {
-      if (a.id < b.id) {
-        return -1;
-      }
-      return a.id > b.id ? 1 : 0;
-    });
-    // All existing records are replaced by the version from the server
-    // and deleted records are removed.
-    for (let i = 0; i < allRecords.length; /* no increment! */) {
-      const rec = allRecords[i];
-      const next = allRecords[i + 1];
-      if ((next && rec.id == next.id) || rec.deleted) {
-        allRecords.splice(i, 1); // remove local record
-      } else {
-        i++;
-      }
-    }
-    const toSerialize = {
-      last_modified: "" + timestamp,
-      data: allRecords,
-    };
-    return CanonicalJSON.stringify(toSerialize, jsesc);
-  },
-
-  /**
-   * If present, import the JSON file into the Remote Settings IndexedDB
-   * for the specified bucket and collection.
-   * (eg. blocklists/certificates, main/onboarding)
-   * @param {String} bucket
-   * @param {String} collection
-   */
-  async importJSONDump(bucket, collection) {
-    const { data: records } = await loadJSONDump(bucket, collection);
-    if (records.length > 0) {
-      await importDumpIDB(bucket, collection, records);
-    }
-    return records.length;
-  },
-};
-
-/**
- * Wrap worker invocations in order to return the `callbackId` along
- * the result. This will allow to transform the worker invocations
- * into promises in `RemoteSettingsWorker.jsm`.
- */
-self.onmessage = (event) => {
-  const { callbackId, method, args = [] } = event.data;
-  Agent[method](...args)
-    .then((result) => {
-      self.postMessage({ callbackId, result });
-    })
-    .catch(error => {
-      console.log(`RemoteSettingsWorker error: ${error}`);
-      self.postMessage({ callbackId, error: "" + error });
-    });
-};
-
-/**
- * Load (from disk) the JSON file distributed with the release for this collection.
- * @param {String}  bucket
- * @param {String}  collection
- */
-async function loadJSONDump(bucket, collection) {
-  const fileURI = `resource://app/defaults/settings/${bucket}/${collection}.json`;
-  let response;
-  try {
-    response = await fetch(fileURI);
-  } catch (e) {
-    // Return empty dataset if file is missing.
-    return { data: [] };
-  }
-  // Will throw if JSON is invalid.
-  return response.json();
-}
-
-/**
- * Import the records into the Remote Settings Chrome IndexedDB.
- *
- * Note: This duplicates some logics from `kinto-offline-client.js`.
- *
- * @param {String} bucket
- * @param {String} collection
- * @param {Array<Object>} records
- */
-async function importDumpIDB(bucket, collection, records) {
-  // Open the DB. It will exist since if we are running this, it means
-  // we already tried to read the timestamp in `remote-settings.js`
-  const db = await openIDB(IDB_NAME, IDB_VERSION);
-
-  // Each entry of the dump will be stored in the records store.
-  // They are indexed by `_cid`, and their status is `synced`.
-  const cid = bucket + "/" + collection;
-  await executeIDB(db, IDB_RECORDS_STORE, store => {
-    // Chain the put operations together, the last one will be waited by
-    // the `transaction.oncomplete` callback.
-    let i = 0;
-    putNext();
-
-    function putNext() {
-      if (i == records.length) {
-        return;
-      }
-      const entry = { ...records[i], _status: "synced", _cid: cid };
-      store.put(entry).onsuccess = putNext; // On error, `transaction.onerror` is called.
-      ++i;
-    }
-  });
-
-  // Store the highest timestamp as the collection timestamp.
-  const timestamp = Math.max(...records.map(record => record.last_modified));
-  await executeIDB(db, IDB_TIMESTAMPS_STORE, store => store.put({ cid, value: timestamp }));
-}
-
-/**
- * Helper to wrap indexedDB.open() into a promise.
- */
-async function openIDB(dbname, version) {
-  return new Promise((resolve, reject) => {
-    const request = indexedDB.open(dbname, version);
-    request.onupgradeneeded = () => {
-      // We should never have to initialize the DB here.
-      reject(new Error(`Error accessing ${dbname} Chrome IDB at version ${version}`));
-    };
-    request.onerror = event => reject(event.target.error);
-    request.onsuccess = event => {
-      const db = event.target.result;
-      resolve(db);
-    };
-  });
-}
-
-/**
- * Helper to wrap some IDBObjectStore operations into a promise.
- *
- * @param {IDBDatabase} db
- * @param {String} storeName
- * @param {function} callback
- */
-async function executeIDB(db, storeName, callback) {
-  const mode = "readwrite";
-  return new Promise((resolve, reject) => {
-    const transaction = db.transaction([storeName], mode);
-    const store = transaction.objectStore(storeName);
-    let result;
-    try {
-      result = callback(store);
-    } catch (e) {
-      transaction.abort();
-      reject(e);
-    }
-    transaction.onerror = event => reject(event.target.error);
-    transaction.oncomplete = event => resolve(result);
-  });
-}
deleted file mode 100644
--- a/services/settings/RemoteSettingsWorker.jsm
+++ /dev/null
@@ -1,53 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-"use strict";
-
-/**
- * Interface to a dedicated thread handling for Remote Settings heavy operations.
- */
-
-// ChromeUtils.import("resource://gre/modules/PromiseWorker.jsm", this);
-
-var EXPORTED_SYMBOLS = ["RemoteSettingsWorker"];
-
-class Worker {
-
-  constructor(source) {
-    this.worker = new ChromeWorker(source);
-    this.worker.onmessage = this._onWorkerMessage.bind(this);
-
-    this.callbacks = new Map();
-    this.lastCallbackId = 0;
-  }
-
-  async _execute(method, args = []) {
-    return new Promise(async (resolve, reject) => {
-      const callbackId = ++this.lastCallbackId;
-      this.callbacks.set(callbackId, [resolve, reject]);
-      this.worker.postMessage({ callbackId, method, args });
-    });
-  }
-
-  _onWorkerMessage(event) {
-    const { callbackId, result, error } = event.data;
-    const [resolve, reject] = this.callbacks.get(callbackId);
-    if (error) {
-      reject(new Error(error));
-    } else {
-      resolve(result);
-    }
-    this.callbacks.delete(callbackId);
-  }
-
-  async canonicalStringify(localRecords, remoteRecords, timestamp) {
-    return this._execute("canonicalStringify", [localRecords, remoteRecords, timestamp]);
-  }
-
-  async importJSONDump(bucket, collection) {
-    return this._execute("importJSONDump", [bucket, collection]);
-  }
-}
-
-var RemoteSettingsWorker = new Worker("resource://services-settings/RemoteSettingsWorker.js");
--- a/services/settings/moz.build
+++ b/services/settings/moz.build
@@ -11,13 +11,11 @@ DIRS += [
 
 EXTRA_COMPONENTS += [
     'RemoteSettingsComponents.js',
     'servicesSettings.manifest',
 ]
 
 EXTRA_JS_MODULES['services-settings'] += [
     'remote-settings.js',
-    'RemoteSettingsWorker.js',
-    'RemoteSettingsWorker.jsm',
 ]
 
 XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']
--- a/services/settings/remote-settings.js
+++ b/services/settings/remote-settings.js
@@ -26,35 +26,33 @@ ChromeUtils.defineModuleGetter(this, "Ca
 ChromeUtils.defineModuleGetter(this, "UptakeTelemetry",
                                "resource://services-common/uptake-telemetry.js");
 ChromeUtils.defineModuleGetter(this, "ClientEnvironmentBase",
                                "resource://gre/modules/components-utils/ClientEnvironment.jsm");
 ChromeUtils.defineModuleGetter(this, "FilterExpressions",
                                "resource://gre/modules/components-utils/FilterExpressions.jsm");
 ChromeUtils.defineModuleGetter(this, "pushBroadcastService",
                                "resource://gre/modules/PushBroadcastService.jsm");
-ChromeUtils.defineModuleGetter(this, "RemoteSettingsWorker",
-                               "resource://services-settings/RemoteSettingsWorker.jsm");
 
 const PREF_SETTINGS_DEFAULT_BUCKET     = "services.settings.default_bucket";
 const PREF_SETTINGS_BRANCH             = "services.settings.";
 const PREF_SETTINGS_SERVER             = "server";
 const PREF_SETTINGS_DEFAULT_SIGNER     = "default_signer";
 const PREF_SETTINGS_VERIFY_SIGNATURE   = "verify_signature";
 const PREF_SETTINGS_SERVER_BACKOFF     = "server.backoff";
 const PREF_SETTINGS_CHANGES_PATH       = "changes.path";
 const PREF_SETTINGS_LAST_UPDATE        = "last_update_seconds";
 const PREF_SETTINGS_LAST_ETAG          = "last_etag";
 const PREF_SETTINGS_CLOCK_SKEW_SECONDS = "clock_skew_seconds";
 const PREF_SETTINGS_LOAD_DUMP          = "load_dump";
 
 // Telemetry update source identifier.
 const TELEMETRY_HISTOGRAM_KEY = "settings-changes-monitoring";
 
-const INVALID_SIGNATURE = "Invalid content signature";
+const INVALID_SIGNATURE = "Invalid content/signature";
 const MISSING_SIGNATURE = "Missing signature";
 
 XPCOMUtils.defineLazyGetter(this, "gPrefs", () => {
   return Services.prefs.getBranch(PREF_SETTINGS_BRANCH);
 });
 XPCOMUtils.defineLazyPreferenceGetter(this, "gVerifySignature", PREF_SETTINGS_BRANCH + PREF_SETTINGS_VERIFY_SIGNATURE, true);
 XPCOMUtils.defineLazyPreferenceGetter(this, "gServerURL", PREF_SETTINGS_BRANCH + PREF_SETTINGS_SERVER);
 XPCOMUtils.defineLazyPreferenceGetter(this, "gChangesPath", PREF_SETTINGS_BRANCH + PREF_SETTINGS_CHANGES_PATH);
@@ -103,18 +101,42 @@ async function jexlFilterFunc(entry, env
     };
     result = await FilterExpressions.eval(filter_expression, context);
   } catch (e) {
     Cu.reportError(e);
   }
   return result ? entry : null;
 }
 
+
+function mergeChanges(collection, localRecords, changes) {
+  const records = {};
+  // Local records by id.
+  localRecords.forEach((record) => records[record.id] = collection.cleanLocalFields(record));
+  // All existing records are replaced by the version from the server.
+  changes.forEach((record) => records[record.id] = record);
+
+  return Object.values(records)
+    // Filter out deleted records.
+    .filter((record) => !record.deleted)
+    // Sort list by record id.
+    .sort((a, b) => {
+      if (a.id < b.id) {
+        return -1;
+      }
+      return a.id > b.id ? 1 : 0;
+    });
+}
+
+
 async function fetchCollectionMetadata(remote, collection, expectedTimestamp) {
   const client = new KintoHttpClient(remote);
+  //
+  // XXX: https://github.com/Kinto/kinto-http.js/issues/307
+  //
   const { signature } = await client.bucket(collection.bucket)
                                     .collection(collection.name)
                                     .getData({ query: { _expected: expectedTimestamp }});
   return signature;
 }
 
 async function fetchRemoteCollection(collection, expectedTimestamp) {
   const client = new KintoHttpClient(gServerURL);
@@ -196,16 +218,43 @@ async function fetchLatestChanges(url, l
     if (!isNaN(value)) {
       backoffSeconds = value;
     }
   }
 
   return {changes, currentEtag, serverTimeMillis, backoffSeconds};
 }
 
+/**
+ * Load the the JSON file distributed with the release for this collection.
+ * @param {String}  bucket
+ * @param {String}  collection
+ * @param {Object}  options
+ * @param {boolean} options.ignoreMissing Do not throw an error if the file is missing.
+ */
+async function loadDumpFile(bucket, collection, { ignoreMissing = true } = {}) {
+  const fileURI = `resource://app/defaults/settings/${bucket}/${collection}.json`;
+  let response;
+  try {
+    // Will throw NetworkError is folder/file is missing.
+    response = await fetch(fileURI);
+    if (!response.ok) {
+      throw new Error(`Could not read from '${fileURI}'`);
+    }
+    // Will throw if JSON is invalid.
+    return response.json();
+  } catch (e) {
+    // A missing file is reported as "NetworError" (see Bug 1493709)
+    if (!ignoreMissing || !/NetworkError/.test(e.message)) {
+      throw e;
+    }
+  }
+  return { data: [] };
+}
+
 
 class RemoteSettingsClient {
 
   constructor(collectionName, { bucketNamePref, signerName, filterFunc = jexlFilterFunc, localFields = [], lastCheckTimePref }) {
     this.collectionName = collectionName;
     this.signerName = signerName;
     this.filterFunc = filterFunc;
     this.localFields = localFields;
@@ -298,25 +347,26 @@ class RemoteSettingsClient {
    * @param  {Object} options         The options object.
    * @param  {Object} options.filters Filter the results (default: `{}`).
    * @param  {Object} options.order   The order to apply   (default: `-last_modified`).
    * @return {Promise}
    */
   async get(options = {}) {
     // In Bug 1451031, we will do some jexl filtering to limit the list items
     // whose target is matched.
-    const { filters = {}, order = "" } = options; // not sorted by default.
+    const { filters = {}, order } = options;
     const c = await this.openCollection();
 
     const timestamp = await c.db.getLastModified();
     // If the local database was never synchronized, then we attempt to load
     // a packaged JSON dump.
     if (timestamp == null) {
       try {
-        await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
+        const { data } = await loadDumpFile(this.bucketName, this.collectionName);
+        await c.loadDump(data);
       } catch (e) {
         // Report but return an empty list since there will be no data anyway.
         Cu.reportError(e);
         return [];
       }
     }
 
     const { data } = await c.list({ filters, order });
@@ -343,17 +393,18 @@ class RemoteSettingsClient {
       let collectionLastModified = await collection.db.getLastModified();
 
       // If there is no data currently in the collection, attempt to import
       // initial data from the application defaults.
       // This allows to avoid synchronizing the whole collection content on
       // cold start.
       if (!collectionLastModified && loadDump) {
         try {
-          await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
+          const initialData = await loadDumpFile(this.bucketName, this.collectionName);
+          await collection.loadDump(initialData.data);
           collectionLastModified = await collection.db.getLastModified();
         } catch (e) {
           // Report but go-on.
           Cu.reportError(e);
         }
       }
 
       // If the data is up to date, there's no need to sync. We still need
@@ -362,23 +413,18 @@ class RemoteSettingsClient {
         this._updateLastCheck(serverTime);
         reportStatus = UptakeTelemetry.STATUS.UP_TO_DATE;
         return;
       }
 
       // If there is a `signerName` and collection signing is enforced, add a
       // hook for incoming changes that validates the signature.
       if (this.signerName && gVerifySignature) {
-        collection.hooks["incoming-changes"] = [async (payload, collection) => {
-          await this._validateCollectionSignature(payload.changes,
-                                                  payload.lastModified,
-                                                  collection,
-                                                  { expectedTimestamp });
-          // In case the signature is valid, apply the changes locally.
-          return payload;
+        collection.hooks["incoming-changes"] = [(payload, collection) => {
+          return this._validateCollectionSignature(payload, collection, { expectedTimestamp });
         }];
       }
 
       // Fetch changes from server.
       let syncResult;
       try {
         // Server changes have priority during synchronization.
         const strategy = Kinto.syncStrategy.SERVER_WINS;
@@ -388,37 +434,34 @@ class RemoteSettingsClient {
         syncResult = await collection.sync({ remote: gServerURL, strategy, expectedTimestamp });
         const { ok } = syncResult;
         if (!ok) {
           // Some synchronization conflicts occured.
           reportStatus = UptakeTelemetry.STATUS.CONFLICT_ERROR;
           throw new Error("Sync failed");
         }
       } catch (e) {
-        if (e.message.includes(INVALID_SIGNATURE)) {
+        if (e.message == INVALID_SIGNATURE) {
           // Signature verification failed during synchronzation.
           reportStatus = UptakeTelemetry.STATUS.SIGNATURE_ERROR;
           // if sync fails with a signature error, it's likely that our
           // local data has been modified in some way.
           // We will attempt to fix this by retrieving the whole
           // remote collection.
           const payload = await fetchRemoteCollection(collection, expectedTimestamp);
           try {
-            await this._validateCollectionSignature(payload.data,
-                                                    payload.last_modified,
-                                                    collection,
-                                                    { expectedTimestamp, ignoreLocal: true });
+            await this._validateCollectionSignature(payload, collection, { expectedTimestamp, ignoreLocal: true });
           } catch (e) {
             reportStatus = UptakeTelemetry.STATUS.SIGNATURE_RETRY_ERROR;
             throw e;
           }
 
           // The signature is good (we haven't thrown).
           // Now we will Inspect what we had locally.
-          const { data: oldData } = await collection.list({ order: "" }); // no need to sort.
+          const { data: oldData } = await collection.list();
 
           // We build a sync result as if a diff-based sync was performed.
           syncResult = { created: [], updated: [], deleted: [] };
 
           // If the remote last_modified is newer than the local last_modified,
           // replace the local data
           const localLastModified = await collection.db.getLastModified();
           if (payload.last_modified >= localLastModified) {
@@ -467,17 +510,17 @@ class RemoteSettingsClient {
         );
       // For updates, keep entries whose updated form is matches the target.
       const updatedFilteredIds = new Set(updatedFiltered.map(e => e.id));
       const updated = allUpdated.filter(({ new: { id } }) => updatedFilteredIds.has(id));
 
       // If every changed entry is filtered, we don't even fire the event.
       if (created.length || updated.length || deleted.length) {
         // Read local collection of records (also filtered).
-        const { data: allData } = await collection.list({ order: "" }); // no need to sort.
+        const { data: allData } = await collection.list();
         const current = await this._filterEntries(allData);
         const payload = { data: { current, created, updated, deleted } };
         try {
           await this.emit("sync", payload);
         } catch (e) {
           reportStatus = UptakeTelemetry.STATUS.APPLY_ERROR;
           throw e;
         }
@@ -497,46 +540,54 @@ class RemoteSettingsClient {
       if (reportStatus === null) {
         reportStatus = UptakeTelemetry.STATUS.SUCCESS;
       }
       // Report success/error status to Telemetry.
       UptakeTelemetry.report(this.identifier, reportStatus);
     }
   }
 
-  async _validateCollectionSignature(remoteRecords, timestamp, collection, options = {}) {
-    const { expectedTimestamp, ignoreLocal = false } = options;
+  async _validateCollectionSignature(payload, collection, options = {}) {
+    const { expectedTimestamp, ignoreLocal } = options;
     // this is a content-signature field from an autograph response.
     const signaturePayload = await fetchCollectionMetadata(gServerURL, collection, expectedTimestamp);
     if (!signaturePayload) {
       throw new Error(MISSING_SIGNATURE);
     }
     const {x5u, signature} = signaturePayload;
     const certChainResponse = await fetch(x5u);
     const certChain = await certChainResponse.text();
 
     const verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
                        .createInstance(Ci.nsIContentSignatureVerifier);
 
-    let localRecords = [];
-    if (!ignoreLocal) {
-      const { data } = await collection.list({ order: "" }); // no need to sort.
-      // Local fields are stripped to compute the collection signature (server does not have them).
-      localRecords = data.map(r => collection.cleanLocalFields(r));
+    let toSerialize;
+    if (ignoreLocal) {
+      toSerialize = {
+        last_modified: `${payload.last_modified}`,
+        data: payload.data,
+      };
+    } else {
+      const {data: localRecords} = await collection.list();
+      const records = mergeChanges(collection, localRecords, payload.changes);
+      toSerialize = {
+        last_modified: `${payload.lastModified}`,
+        data: records,
+      };
     }
 
-    const serialized = await RemoteSettingsWorker.canonicalStringify(localRecords,
-                                                                     remoteRecords,
-                                                                     timestamp);
-    if (!verifier.verifyContentSignature(serialized,
-                                         "p384ecdsa=" + signature,
-                                         certChain,
-                                         this.signerName)) {
-      throw new Error(INVALID_SIGNATURE + ` (${collection.bucket}/${collection.name})`);
+    const serialized = CanonicalJSON.stringify(toSerialize);
+
+    if (verifier.verifyContentSignature(serialized, "p384ecdsa=" + signature,
+                                        certChain,
+                                        this.signerName)) {
+      // In case the hash is valid, apply the changes locally.
+      return payload;
     }
+    throw new Error(INVALID_SIGNATURE);
   }
 
   /**
    * Save last time server was checked in users prefs.
    *
    * @param {Date} serverTime   the current date return by server.
    */
   _updateLastCheck(serverTime) {
@@ -572,17 +623,17 @@ async function hasLocalData(client) {
  * Check if we ship a JSON dump for the specified bucket and collection.
  *
  * @param {String} bucket
  * @param {String} collection
  * @return {bool} Whether it is present or not.
  */
 async function hasLocalDump(bucket, collection) {
   try {
-    await fetch(`resource://app/defaults/settings/${bucket}/${collection}.json`);
+    await loadDumpFile(bucket, collection, {ignoreMissing: false});
     return true;
   } catch (e) {
     return false;
   }
 }
 
 
 function remoteSettingsFunction() {
deleted file mode 100644
--- a/services/settings/test/unit/test_remote_settings_worker.js
+++ /dev/null
@@ -1,44 +0,0 @@
-/* import-globals-from ../../../common/tests/unit/head_helpers.js */
-
-ChromeUtils.import("resource://gre/modules/Services.jsm");
-
-const { RemoteSettingsWorker } = ChromeUtils.import("resource://services-settings/RemoteSettingsWorker.jsm", {});
-const { Kinto } = ChromeUtils.import("resource://services-common/kinto-offline-client.js", {});
-
-
-add_task(async function test_canonicaljson_merges_remote_into_local() {
-  const localRecords = [{ id: "1", title: "title 1" }, { id: "2", title: "title 2" }, { id: "3", title: "title 3" }];
-  const remoteRecords = [{ id: "2", title: "title b" }, { id: "3", deleted: true }];
-  const timestamp = 42;
-
-  const serialized = await RemoteSettingsWorker.canonicalStringify(localRecords, remoteRecords, timestamp);
-
-  Assert.equal('{"data":[{"id":"1","title":"title 1"},{"id":"2","title":"title b"}],"last_modified":"42"}', serialized);
-});
-
-
-add_task(async function test_import_json_dump_into_idb() {
-  const kintoCollection = new Kinto({
-    bucket: "main",
-    adapter: Kinto.adapters.IDB,
-    adapterOptions: { dbName: "remote-settings" },
-  }).collection("language-dictionaries");
-  const { data: before } = await kintoCollection.list();
-  Assert.equal(before.length, 0);
-
-  await RemoteSettingsWorker.importJSONDump("main", "language-dictionaries");
-
-  const { data: after } = await kintoCollection.list();
-  Assert.ok(after.length > 0);
-});
-
-
-add_task(async function test_throws_error_if_worker_fails() {
-  let error;
-  try {
-    await RemoteSettingsWorker.canonicalStringify(null, [], 42);
-  } catch (e) {
-    error = e;
-  }
-  Assert.equal("TypeError: localRecords is null; can't access its \"concat\" property", error.message);
-});
--- a/services/settings/test/unit/xpcshell.ini
+++ b/services/settings/test/unit/xpcshell.ini
@@ -1,9 +1,8 @@
 [DEFAULT]
 head = ../../../common/tests/unit/head_global.js ../../../common/tests/unit/head_helpers.js
 firefox-appdir = browser
 tags = remote-settings
 
 [test_remote_settings.js]
 [test_remote_settings_poll.js]
-[test_remote_settings_worker.js]
 [test_remote_settings_jexl_filters.js]
--- a/toolkit/modules/CanonicalJSON.jsm
+++ b/toolkit/modules/CanonicalJSON.jsm
@@ -1,47 +1,46 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 var EXPORTED_SYMBOLS = ["CanonicalJSON"];
 
+ChromeUtils.defineModuleGetter(this, "jsesc",
+                               "resource://gre/modules/third_party/jsesc/jsesc.js");
+
 var CanonicalJSON = {
   /**
    * Return the canonical JSON form of the passed source, sorting all the object
    * keys recursively. Note that this method will cause an infinite loop if
    * cycles exist in the source (bug 1265357).
    *
    * @param source
    *        The elements to be serialized.
    *
    * The output will have all unicode chars escaped with the unicode codepoint
    * as lowercase hexadecimal.
    *
    * @usage
    *        CanonicalJSON.stringify(listOfRecords);
    **/
-  stringify: function stringify(source, jsescFn) {
-    if (typeof jsescFn != "function") {
-      const { jsesc } = ChromeUtils.import("resource://gre/modules/third_party/jsesc/jsesc.js", {});
-      jsescFn = jsesc;
-    }
+  stringify: function stringify(source) {
     if (Array.isArray(source)) {
       const jsonArray = source.map(x => typeof x === "undefined" ? null : x);
-      return "[" + jsonArray.map(item => stringify(item, jsescFn)).join(",") + "]";
+      return `[${jsonArray.map(stringify).join(",")}]`;
     }
 
     if (typeof source === "number") {
       if (source === 0) {
         return (Object.is(source, -0)) ? "-0" : "0";
       }
     }
 
     // Leverage jsesc library, mainly for unicode escaping.
-    const toJSON = (input) => jsescFn(input, {lowercaseHex: true, json: true});
+    const toJSON = (input) => jsesc(input, {lowercaseHex: true, json: true});
 
     if (typeof source !== "object" || source === null) {
       return toJSON(source);
     }
 
     // Dealing with objects, ordering keys.
     const sortedKeys = Object.keys(source).sort();
     const lastIndex = sortedKeys.length - 1;
@@ -49,12 +48,12 @@ var CanonicalJSON = {
       const value = source[key];
       // JSON.stringify drops keys with an undefined value.
       if (typeof value === "undefined") {
         return serial;
       }
       const jsonValue = value && value.toJSON ? value.toJSON() : value;
       const suffix = index !== lastIndex ? "," : "";
       const escapedKey = toJSON(key);
-      return serial + escapedKey + ":" + stringify(jsonValue, jsescFn) + suffix;
+      return serial + `${escapedKey}:${stringify(jsonValue)}${suffix}`;
     }, "{") + "}";
   },
 };
--- a/tools/lint/eslint/modules.json
+++ b/tools/lint/eslint/modules.json
@@ -18,17 +18,16 @@
   "bogus_element_type.jsm": [],
   "bookmark_repair.js": ["BookmarkRepairRequestor", "BookmarkRepairResponder"],
   "bookmark_validator.js": ["BookmarkValidator", "BookmarkProblemData"],
   "bookmarks.js": ["BookmarksEngine", "PlacesItem", "Bookmark", "BookmarkFolder", "BookmarkQuery", "Livemark", "BookmarkSeparator", "BufferedBookmarksEngine"],
   "bookmarks.jsm": ["PlacesItem", "Bookmark", "Separator", "Livemark", "BookmarkFolder", "DumpBookmarks"],
   "BootstrapMonitor.jsm": ["monitor"],
   "browser-loader.js": ["BrowserLoader"],
   "browserid_identity.js": ["BrowserIDManager", "AuthenticationError"],
-  "CanonicalJSON.jsm": ["CanonicalJSON"],
   "CertUtils.jsm": ["CertUtils"],
   "clients.js": ["ClientEngine", "ClientsRec"],
   "collection_repair.js": ["getRepairRequestor", "getAllRepairRequestors", "CollectionRepairRequestor", "getRepairResponder", "CollectionRepairResponder"],
   "collection_validator.js": ["CollectionValidator", "CollectionProblemData"],
   "Console.jsm": ["console", "ConsoleAPI"],
   "constants.js": ["WEAVE_VERSION", "SYNC_API_VERSION", "STORAGE_VERSION", "PREFS_BRANCH", "DEFAULT_KEYBUNDLE_NAME", "SYNC_KEY_ENCODED_LENGTH", "SYNC_KEY_DECODED_LENGTH", "NO_SYNC_NODE_INTERVAL", "MAX_ERROR_COUNT_BEFORE_BACKOFF", "MINIMUM_BACKOFF_INTERVAL", "MAXIMUM_BACKOFF_INTERVAL", "HMAC_EVENT_INTERVAL", "MASTER_PASSWORD_LOCKED_RETRY_INTERVAL", "DEFAULT_GUID_FETCH_BATCH_SIZE", "DEFAULT_DOWNLOAD_BATCH_SIZE", "SINGLE_USER_THRESHOLD", "MULTI_DEVICE_THRESHOLD", "SCORE_INCREMENT_SMALL", "SCORE_INCREMENT_MEDIUM", "SCORE_INCREMENT_XLARGE", "SCORE_UPDATE_DELAY", "IDLE_OBSERVER_BACK_DELAY", "URI_LENGTH_MAX", "MAX_HISTORY_UPLOAD", "MAX_HISTORY_DOWNLOAD", "STATUS_OK", "SYNC_FAILED", "LOGIN_FAILED", "SYNC_FAILED_PARTIAL", "CLIENT_NOT_CONFIGURED", "STATUS_DISABLED", "MASTER_PASSWORD_LOCKED", "LOGIN_SUCCEEDED", "SYNC_SUCCEEDED", "ENGINE_SUCCEEDED", "LOGIN_FAILED_NO_USERNAME", "LOGIN_FAILED_NO_PASSPHRASE", "LOGIN_FAILED_NETWORK_ERROR", "LOGIN_FAILED_SERVER_ERROR", "LOGIN_FAILED_INVALID_PASSPHRASE", "LOGIN_FAILED_LOGIN_REJECTED", "METARECORD_DOWNLOAD_FAIL", "VERSION_OUT_OF_DATE", "CREDENTIALS_CHANGED", "ABORT_SYNC_COMMAND", "NO_SYNC_NODE_FOUND", "OVER_QUOTA", "SERVER_MAINTENANCE", "RESPONSE_OVER_QUOTA", "ENGINE_UPLOAD_FAIL", "ENGINE_DOWNLOAD_FAIL", "ENGINE_UNKNOWN_FAIL", "ENGINE_APPLY_FAIL", "ENGINE_BATCH_INTERRUPTED", "kSyncMasterPasswordLocked", "kSyncWeaveDisabled", "kSyncNetworkOffline", "kSyncBackoffNotMet", "kFirstSyncChoiceNotMade", "kSyncNotConfigured", "kFirefoxShuttingDown", "DEVICE_TYPE_DESKTOP", "DEVICE_TYPE_MOBILE", "SQLITE_MAX_VARIABLE_NUMBER"],
   "Constants.jsm": ["Roles", "Events", "Relations", "Filters", "States", "Prefilters"],
   "ContactDB.jsm": ["ContactDB", "DB_NAME", "STORE_NAME", "SAVED_GETALL_STORE_NAME", "REVISION_STORE", "DB_VERSION"],
@@ -96,17 +95,16 @@
   "history.jsm": ["HistoryEntry", "DumpHistory"],
   "Http.jsm": ["httpRequest", "percentEncode"],
   "httpd.js": ["HTTP_400", "HTTP_401", "HTTP_402", "HTTP_403", "HTTP_404", "HTTP_405", "HTTP_406", "HTTP_407", "HTTP_408", "HTTP_409", "HTTP_410", "HTTP_411", "HTTP_412", "HTTP_413", "HTTP_414", "HTTP_415", "HTTP_417", "HTTP_500", "HTTP_501", "HTTP_502", "HTTP_503", "HTTP_504", "HTTP_505", "HttpError", "HttpServer"],
   "import_module.jsm": ["MODULE_IMPORTED", "MODULE_URI", "SUBMODULE_IMPORTED", "same_scope", "SUBMODULE_IMPORTED_TO_SCOPE"],
   "import_sub_module.jsm": ["SUBMODULE_IMPORTED", "test_obj"],
   "InlineSpellChecker.jsm": ["InlineSpellChecker", "SpellCheckHelper"],
   "JSDOMParser.js": ["JSDOMParser"],
   "jsdebugger.jsm": ["addDebuggerToGlobal"],
-  "jsesc.js": ["jsesc"],
   "json2.js": ["JSON"],
   "keys.js": ["BulkKeyBundle", "SyncKeyBundle"],
   "KeyValueParser.jsm": ["parseKeyValuePairsFromLines", "parseKeyValuePairs", "parseKeyValuePairsFromFile", "parseKeyValuePairsFromFileAsync"],
   "kinto-http-client.js": ["KintoHttpClient"],
   "kinto-offline-client.js": ["Kinto"],
   "kinto-storage-adapter.js": ["FirefoxAdapter"],
   "L10nRegistry.jsm": ["L10nRegistry", "FileSource", "IndexedFileSource"],
   "loader-plugin-raw.jsm": ["requireRawId"],