Bug 1253740 - Introduce extension-storage engine with a sanity test, r=markh
☠☠ backed out by 64f9cf455380 ☠ ☠
authorEthan Glasser-Camp <eglassercamp@mozilla.com>
Thu, 08 Sep 2016 14:23:12 -0400
changeset 320684 39e08d903b4898af0f50bd9e96e301b6cd44cbc0
parent 320683 3963331638977f8ed025f85a159dac995480a1a8
child 320685 77be2faa619f828b9e6767c822561bc125b265ce
push id83436
push userkwierso@gmail.com
push dateThu, 03 Nov 2016 00:31:38 +0000
treeherdermozilla-inbound@227bccbfea15 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmarkh
bugs1253740
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1253740 - Introduce extension-storage engine with a sanity test, r=markh Note that this "enables" the engine using a pref, even though it might not be ready yet, so that the tests can pass. MozReview-Commit-ID: AZ0TVERiQDU
browser/app/profile/firefox.js
services/sync/modules/engines/extension-storage.js
services/sync/modules/service.js
services/sync/modules/telemetry.js
services/sync/moz.build
services/sync/services-sync.js
services/sync/tests/unit/head_helpers.js
services/sync/tests/unit/test_extension_storage_engine.js
services/sync/tests/unit/test_extension_storage_tracker.js
services/sync/tests/unit/test_load_modules.js
services/sync/tests/unit/xpcshell.ini
toolkit/components/extensions/ExtensionStorageSync.jsm
toolkit/components/extensions/test/xpcshell/head_sync.js
toolkit/components/extensions/test/xpcshell/xpcshell.ini
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1038,17 +1038,17 @@ pref("browser.taskbar.lists.enabled", tr
 pref("browser.taskbar.lists.frequent.enabled", true);
 pref("browser.taskbar.lists.recent.enabled", false);
 pref("browser.taskbar.lists.maxListItemCount", 7);
 pref("browser.taskbar.lists.tasks.enabled", true);
 pref("browser.taskbar.lists.refreshInSeconds", 120);
 #endif
 
 // The sync engines to use.
-pref("services.sync.registerEngines", "Bookmarks,Form,History,Password,Prefs,Tab,Addons");
+pref("services.sync.registerEngines", "Bookmarks,Form,History,Password,Prefs,Tab,Addons,ExtensionStorage");
 // Preferences to be synced by default
 pref("services.sync.prefs.sync.accessibility.blockautorefresh", true);
 pref("services.sync.prefs.sync.accessibility.browsewithcaret", true);
 pref("services.sync.prefs.sync.accessibility.typeaheadfind", true);
 pref("services.sync.prefs.sync.accessibility.typeaheadfind.linksonly", true);
 pref("services.sync.prefs.sync.addons.ignoreUserEnabledChanges", true);
 // The addons prefs related to repository verification are intentionally
 // not synced for security reasons. If a system is compromised, a user
new file mode 100644
--- /dev/null
+++ b/services/sync/modules/engines/extension-storage.js
@@ -0,0 +1,103 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ['ExtensionStorageEngine'];
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorageSync",
+                                  "resource://gre/modules/ExtensionStorageSync.jsm");
+
+/**
+ * The Engine that manages syncing for the web extension "storage"
+ * API, and in particular ext.storage.sync.
+ *
+ * ext.storage.sync is implemented using Kinto, so it has mechanisms
+ * for syncing that we do not need to integrate in the Firefox Sync
+ * framework, so this is something of a stub.
+ */
+this.ExtensionStorageEngine = function ExtensionStorageEngine(service) {
+  SyncEngine.call(this, "Extension-Storage", service);
+};
+ExtensionStorageEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _trackerObj: ExtensionStorageTracker,
+  // we don't need these since we implement our own sync logic
+  _storeObj: undefined,
+  _recordObj: undefined,
+
+  syncPriority: 10,
+
+  _sync: function () {
+    return Async.promiseSpinningly(ExtensionStorageSync.syncAll());
+  },
+
+  get enabled() {
+    // By default, we sync extension storage if we sync addons. This
+    // lets us simplify the UX since users probably don't consider
+    // "extension preferences" a separate category of syncing.
+    // However, we also respect engine.extension-storage.force, which
+    // can be set to true or false, if a power user wants to customize
+    // the behavior despite the lack of UI.
+    const forced = Svc.Prefs.get("engine." + this.prefName + ".force", undefined);
+    if (forced !== undefined) {
+      return forced;
+    }
+    return Svc.Prefs.get("engine.addons", false);
+  },
+};
+
+function ExtensionStorageTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+ExtensionStorageTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  startTracking: function () {
+    Svc.Obs.add("ext.storage.sync-changed", this);
+  },
+
+  stopTracking: function () {
+    Svc.Obs.remove("ext.storage.sync-changed", this);
+  },
+
+  observe: function (subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+
+    if (this.ignoreAll) {
+      return;
+    }
+
+    if (topic !== "ext.storage.sync-changed") {
+      return;
+    }
+
+    // Single adds, removes and changes are not so important on their
+    // own, so let's just increment score a bit.
+    this.score += SCORE_INCREMENT_MEDIUM;
+  },
+
+  // Override a bunch of methods which don't do anything for us.
+  // This is a performance hack.
+  saveChangedIDs: function() {
+  },
+  loadChangedIDs: function() {
+  },
+  ignoreID: function() {
+  },
+  unignoreID: function() {
+  },
+  addChangedID: function() {
+  },
+  removeChangedID: function() {
+  },
+  clearChangedIDs: function() {
+  },
+};
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -39,16 +39,17 @@ Cu.import("resource://services-sync/util
 const ENGINE_MODULES = {
   Addons: "addons.js",
   Bookmarks: "bookmarks.js",
   Form: "forms.js",
   History: "history.js",
   Password: "passwords.js",
   Prefs: "prefs.js",
   Tab: "tabs.js",
+  ExtensionStorage: "extension-storage.js",
 };
 
 const STORAGE_INFO_TYPES = [INFO_COLLECTIONS,
                             INFO_COLLECTION_USAGE,
                             INFO_COLLECTION_COUNTS,
                             INFO_QUOTA];
 
 function Sync11Service() {
--- a/services/sync/modules/telemetry.js
+++ b/services/sync/modules/telemetry.js
@@ -46,17 +46,17 @@ const TOPICS = [
   "weave:engine:validate:finish",
   "weave:engine:validate:error",
 ];
 
 const PING_FORMAT_VERSION = 1;
 
 // The set of engines we record telemetry for - any other engines are ignored.
 const ENGINES = new Set(["addons", "bookmarks", "clients", "forms", "history",
-                         "passwords", "prefs", "tabs"]);
+                         "passwords", "prefs", "tabs", "extension-storage"]);
 
 // A regex we can use to replace the profile dir in error messages. We use a
 // regexp so we can simply replace all case-insensitive occurences.
 // This escaping function is from:
 // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions
 const reProfileDir = new RegExp(
         OS.Constants.Path.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"),
         "gi");
--- a/services/sync/moz.build
+++ b/services/sync/moz.build
@@ -47,16 +47,17 @@ EXTRA_PP_JS_MODULES['services-sync'] += 
 # Definitions used by constants.js
 DEFINES['weave_version'] = '1.54.0'
 DEFINES['weave_id'] = '{340c2bbc-ce74-4362-90b5-7c26312808ef}'
 
 EXTRA_JS_MODULES['services-sync'].engines += [
     'modules/engines/addons.js',
     'modules/engines/bookmarks.js',
     'modules/engines/clients.js',
+    'modules/engines/extension-storage.js',
     'modules/engines/forms.js',
     'modules/engines/history.js',
     'modules/engines/passwords.js',
     'modules/engines/prefs.js',
     'modules/engines/tabs.js',
 ]
 
 EXTRA_JS_MODULES['services-sync'].stages += [
--- a/services/sync/services-sync.js
+++ b/services/sync/services-sync.js
@@ -26,16 +26,17 @@ pref("services.sync.errorhandler.network
 
 pref("services.sync.engine.addons", true);
 pref("services.sync.engine.bookmarks", true);
 pref("services.sync.engine.history", true);
 pref("services.sync.engine.passwords", true);
 pref("services.sync.engine.prefs", true);
 pref("services.sync.engine.tabs", true);
 pref("services.sync.engine.tabs.filteredUrls", "^(about:.*|chrome://weave/.*|wyciwyg:.*|file:.*|blob:.*)$");
+pref("services.sync.engine.extension-storage", true);
 
 pref("services.sync.jpake.serverURL", "https://setup.services.mozilla.com/");
 pref("services.sync.jpake.pollInterval", 1000);
 pref("services.sync.jpake.firstMsgMaxTries", 300); // 5 minutes
 pref("services.sync.jpake.lastMsgMaxTries", 300);  // 5 minutes
 pref("services.sync.jpake.maxTries", 10);
 
 // If true, add-on sync ignores changes to the user-enabled flag. This
@@ -63,16 +64,17 @@ pref("services.sync.log.logger.service.j
 pref("services.sync.log.logger.engine.bookmarks", "Debug");
 pref("services.sync.log.logger.engine.clients", "Debug");
 pref("services.sync.log.logger.engine.forms", "Debug");
 pref("services.sync.log.logger.engine.history", "Debug");
 pref("services.sync.log.logger.engine.passwords", "Debug");
 pref("services.sync.log.logger.engine.prefs", "Debug");
 pref("services.sync.log.logger.engine.tabs", "Debug");
 pref("services.sync.log.logger.engine.addons", "Debug");
+pref("services.sync.log.logger.engine.extension-storage", "Debug");
 pref("services.sync.log.logger.engine.apps", "Debug");
 pref("services.sync.log.logger.identity", "Debug");
 pref("services.sync.log.logger.userapi", "Debug");
 pref("services.sync.log.cryptoDebug", false);
 
 pref("services.sync.fxa.termsURL", "https://accounts.firefox.com/legal/terms");
 pref("services.sync.fxa.privacyURL", "https://accounts.firefox.com/legal/privacy");
 
--- a/services/sync/tests/unit/head_helpers.js
+++ b/services/sync/tests/unit/head_helpers.js
@@ -71,16 +71,34 @@ function ExtensionsTestPath(path) {
 function loadAddonTestFunctions() {
   const path = ExtensionsTestPath("/head_addons.js");
   let file = do_get_file(path);
   let uri = Services.io.newFileURI(file);
   Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
 }
 
+function webExtensionsTestPath(path) {
+  if (path[0] != "/") {
+    throw Error("Path must begin with '/': " + path);
+  }
+
+  return "../../../../toolkit/components/extensions/test/xpcshell" + path;
+}
+
+/**
+ * Loads the WebExtension test functions by importing its test file.
+ */
+function loadWebExtensionTestFunctions() {
+  const path = webExtensionsTestPath("/head_sync.js");
+  let file = do_get_file(path);
+  let uri = Services.io.newFileURI(file);
+  Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
+}
+
 function getAddonInstall(name) {
   let f = do_get_file(ExtensionsTestPath("/addons/" + name + ".xpi"));
   let cb = Async.makeSyncCallback();
   AddonManager.getInstallForFile(f, cb);
 
   return Async.waitForSyncCallback(cb);
 }
 
new file mode 100644
--- /dev/null
+++ b/services/sync/tests/unit/test_extension_storage_engine.js
@@ -0,0 +1,62 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/extension-storage.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://testing-common/services/sync/utils.js");
+Cu.import("resource://gre/modules/ExtensionStorageSync.jsm");
+
+Service.engineManager.register(ExtensionStorageEngine);
+const engine = Service.engineManager.get("extension-storage");
+do_get_profile();   // so we can use FxAccounts
+loadWebExtensionTestFunctions();
+
+function mock(options) {
+  let calls = [];
+  let ret = function() {
+    calls.push(arguments);
+    return options.returns;
+  }
+  Object.setPrototypeOf(ret, {
+    __proto__: Function.prototype,
+    get calls() {
+      return calls;
+    }
+  });
+  return ret;
+}
+
+add_task(function* test_calling_sync_calls__sync() {
+  let oldSync = ExtensionStorageEngine.prototype._sync;
+  let syncMock = ExtensionStorageEngine.prototype._sync = mock({returns: true});
+  try {
+    // I wanted to call the main sync entry point for the entire
+    // package, but that fails because it tries to sync ClientEngine
+    // first, which fails.
+    yield engine.sync();
+  } finally {
+    ExtensionStorageEngine.prototype._sync = oldSync;
+  }
+  equal(syncMock.calls.length, 1);
+});
+
+add_task(function* test_calling_sync_calls_ext_storage_sync() {
+  const extension = {id: "my-extension"};
+  let oldSync = ExtensionStorageSync.syncAll;
+  let syncMock = ExtensionStorageSync.syncAll = mock({returns: Promise.resolve()});
+  try {
+    yield* withSyncContext(function* (context) {
+      // Set something so that everyone knows that we're using storage.sync
+      yield ExtensionStorageSync.set(extension, {"a": "b"}, context);
+
+      yield engine._sync();
+    });
+  } finally {
+    ExtensionStorageSync.syncAll = oldSync;
+  }
+  do_check_true(syncMock.calls.length >= 1);
+});
new file mode 100644
--- /dev/null
+++ b/services/sync/tests/unit/test_extension_storage_tracker.js
@@ -0,0 +1,38 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/extension-storage.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://gre/modules/ExtensionStorageSync.jsm");
+
+Service.engineManager.register(ExtensionStorageEngine);
+const engine = Service.engineManager.get("extension-storage");
+do_get_profile();   // so we can use FxAccounts
+loadWebExtensionTestFunctions();
+
+add_task(function* test_changing_extension_storage_changes_score() {
+  const tracker = engine._tracker;
+  const extension = {id: "my-extension-id"};
+  Svc.Obs.notify("weave:engine:start-tracking");
+  yield* withSyncContext(function*(context) {
+    yield ExtensionStorageSync.set(extension, {"a": "b"}, context);
+  });
+  do_check_eq(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  tracker.resetScore();
+  yield* withSyncContext(function*(context) {
+    yield ExtensionStorageSync.remove(extension, "a", context);
+  });
+  do_check_eq(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  Svc.Obs.notify("weave:engine:stop-tracking");
+});
+
+function run_test() {
+  run_next_test();
+}
--- a/services/sync/tests/unit/test_load_modules.js
+++ b/services/sync/tests/unit/test_load_modules.js
@@ -4,16 +4,17 @@
 const modules = [
   "addonutils.js",
   "addonsreconciler.js",
   "browserid_identity.js",
   "constants.js",
   "engines/addons.js",
   "engines/bookmarks.js",
   "engines/clients.js",
+  "engines/extension-storage.js",
   "engines/forms.js",
   "engines/history.js",
   "engines/passwords.js",
   "engines/prefs.js",
   "engines/tabs.js",
   "engines.js",
   "identity.js",
   "jpakeclient.js",
--- a/services/sync/tests/unit/xpcshell.ini
+++ b/services/sync/tests/unit/xpcshell.ini
@@ -10,16 +10,17 @@ support-files =
   missing-sourceuri.xml
   missing-xpi-search.xml
   places_v10_from_v11.sqlite
   rewrite-search.xml
   sync_ping_schema.json
   systemaddon-search.xml
   !/services/common/tests/unit/head_helpers.js
   !/toolkit/mozapps/extensions/test/xpcshell/head_addons.js
+  !/toolkit/components/extensions/test/xpcshell/head_sync.js
 
 # The manifest is roughly ordered from low-level to high-level. When making
 # systemic sweeping changes, this makes it easier to identify errors closer to
 # the source.
 
 # Ensure we can import everything.
 [test_load_modules.js]
 
@@ -157,16 +158,18 @@ tags = addons
 [test_bookmark_smart_bookmarks.js]
 [test_bookmark_store.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)
 skip-if = debug
 [test_bookmark_tracker.js]
 [test_bookmark_validator.js]
 [test_clients_engine.js]
 [test_clients_escape.js]
+[test_extension_storage_engine.js]
+[test_extension_storage_tracker.js]
 [test_forms_store.js]
 [test_forms_tracker.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)
 skip-if = debug
 [test_history_engine.js]
 [test_history_store.js]
 [test_history_tracker.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)
--- a/toolkit/components/extensions/ExtensionStorageSync.jsm
+++ b/toolkit/components/extensions/ExtensionStorageSync.jsm
@@ -20,16 +20,18 @@ const {
 } = Cu.import("resource://gre/modules/ExtensionUtils.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "AppsUtils",
                                   "resource://gre/modules/AppsUtils.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorage",
                                   "resource://gre/modules/ExtensionStorage.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "loadKinto",
                                   "resource://services-common/kinto-offline-client.js");
+XPCOMUtils.defineLazyModuleGetter(this, "Observers",
+                                  "resource://services-common/observers.js");
 XPCOMUtils.defineLazyModuleGetter(this, "Task",
                                   "resource://gre/modules/Task.jsm");
 XPCOMUtils.defineLazyPreferenceGetter(this, "prefPermitsStorageSync",
                                       STORAGE_SYNC_ENABLED_PREF, false);
 
 /* globals prefPermitsStorageSync */
 
 // Map of Extensions to Promise<Collections>.
@@ -322,16 +324,17 @@ this.ExtensionStorageSync = {
     let listeners = this.listeners.get(extension);
     listeners.delete(listener);
     if (listeners.size == 0) {
       this.listeners.delete(extension);
     }
   },
 
   notifyListeners(extension, changes) {
+    Observers.notify("ext.storage.sync-changed");
     let listeners = this.listeners.get(extension) || new Set();
     if (listeners) {
       for (let listener of listeners) {
         runSafeSyncWithoutClone(listener, changes);
       }
     }
   },
 };
new file mode 100644
--- /dev/null
+++ b/toolkit/components/extensions/test/xpcshell/head_sync.js
@@ -0,0 +1,67 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/* exported withSyncContext */
+
+Components.utils.import("resource://gre/modules/Services.jsm", this);
+Components.utils.import("resource://gre/modules/ExtensionUtils.jsm", this);
+
+var {
+  BaseContext,
+} = ExtensionUtils;
+
+class Context extends BaseContext {
+  constructor(principal) {
+    super();
+    Object.defineProperty(this, "principal", {
+      value: principal,
+      configurable: true,
+    });
+    this.sandbox = Components.utils.Sandbox(principal, {wantXrays: false});
+    this.extension = {id: "test@web.extension"};
+  }
+
+  get cloneScope() {
+    return this.sandbox;
+  }
+}
+
+/**
+ * Call the given function with a newly-constructed context.
+ * Unload the context on the way out.
+ *
+ * @param {function} f    the function to call
+ */
+function* withContext(f) {
+  const ssm = Services.scriptSecurityManager;
+  const PRINCIPAL1 = ssm.createCodebasePrincipalFromOrigin("http://www.example.org");
+  const context = new Context(PRINCIPAL1);
+  try {
+    yield* f(context);
+  } finally {
+    yield context.unload();
+  }
+}
+
+/**
+ * Like withContext(), but also turn on the "storage.sync" pref for
+ * the duration of the function.
+ * Calls to this function can be replaced with calls to withContext
+ * once the pref becomes on by default.
+ *
+ * @param {function} f    the function to call
+ */
+function* withSyncContext(f) {
+  const STORAGE_SYNC_PREF = "webextensions.storage.sync.enabled";
+  let prefs = Services.prefs;
+
+  try {
+    prefs.setBoolPref(STORAGE_SYNC_PREF, true);
+    yield* withContext(f);
+  } finally {
+    prefs.clearUserPref(STORAGE_SYNC_PREF);
+  }
+}
--- a/toolkit/components/extensions/test/xpcshell/xpcshell.ini
+++ b/toolkit/components/extensions/test/xpcshell/xpcshell.ini
@@ -1,15 +1,15 @@
 [DEFAULT]
 head = head.js
 tail =
 firefox-appdir = browser
 skip-if = appname == "thunderbird"
 support-files =
-  data/**
+  data/** head_sync.js
 tags = webextensions
 
 [test_csp_custom_policies.js]
 [test_csp_validator.js]
 [test_ext_alarms.js]
 [test_ext_alarms_does_not_fire.js]
 [test_ext_alarms_periodic.js]
 [test_ext_alarms_replaces.js]