Bug 1253740 - Introduce extension-storage engine with a sanity test, r=markh
authorEthan Glasser-Camp <eglassercamp@mozilla.com>
Thu, 08 Sep 2016 14:23:12 -0400
changeset 322350 6e838e5593215d92588dbcb34bd83b6b3d5e8fd2
parent 322349 bbfdfc4befb49e3756707a277019a0a48bc88ba6
child 322351 bd3541139b99a84b616bd75c67d552ba3eb63ed1
push id83844
push usercbook@mozilla.com
push dateMon, 14 Nov 2016 09:35:48 +0000
treeherdermozilla-inbound@c29cedd0d3f8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmarkh
bugs1253740
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1253740 - Introduce extension-storage engine with a sanity test, r=markh Note that this "enables" the engine using a pref, even though it might not be ready yet, so that the tests can pass. MozReview-Commit-ID: AZ0TVERiQDU
browser/app/profile/firefox.js
services/sync/modules/engines/extension-storage.js
services/sync/modules/service.js
services/sync/modules/telemetry.js
services/sync/moz.build
services/sync/services-sync.js
services/sync/tests/unit/head_helpers.js
services/sync/tests/unit/test_extension_storage_engine.js
services/sync/tests/unit/test_extension_storage_tracker.js
services/sync/tests/unit/test_load_modules.js
services/sync/tests/unit/xpcshell.ini
toolkit/components/extensions/ExtensionStorageSync.jsm
toolkit/components/extensions/test/xpcshell/head_sync.js
toolkit/components/extensions/test/xpcshell/xpcshell.ini
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1038,17 +1038,17 @@ pref("browser.taskbar.lists.enabled", tr
 pref("browser.taskbar.lists.frequent.enabled", true);
 pref("browser.taskbar.lists.recent.enabled", false);
 pref("browser.taskbar.lists.maxListItemCount", 7);
 pref("browser.taskbar.lists.tasks.enabled", true);
 pref("browser.taskbar.lists.refreshInSeconds", 120);
 #endif
 
 // The sync engines to use.
-pref("services.sync.registerEngines", "Bookmarks,Form,History,Password,Prefs,Tab,Addons");
+pref("services.sync.registerEngines", "Bookmarks,Form,History,Password,Prefs,Tab,Addons,ExtensionStorage");
 // Preferences to be synced by default
 pref("services.sync.prefs.sync.accessibility.blockautorefresh", true);
 pref("services.sync.prefs.sync.accessibility.browsewithcaret", true);
 pref("services.sync.prefs.sync.accessibility.typeaheadfind", true);
 pref("services.sync.prefs.sync.accessibility.typeaheadfind.linksonly", true);
 pref("services.sync.prefs.sync.addons.ignoreUserEnabledChanges", true);
 // The addons prefs related to repository verification are intentionally
 // not synced for security reasons. If a system is compromised, a user
new file mode 100644
--- /dev/null
+++ b/services/sync/modules/engines/extension-storage.js
@@ -0,0 +1,103 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ['ExtensionStorageEngine'];
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorageSync",
+                                  "resource://gre/modules/ExtensionStorageSync.jsm");
+
+/**
+ * The Engine that manages syncing for the web extension "storage"
+ * API, and in particular ext.storage.sync.
+ *
+ * ext.storage.sync is implemented using Kinto, so it has mechanisms
+ * for syncing that we do not need to integrate in the Firefox Sync
+ * framework, so this is something of a stub.
+ */
+this.ExtensionStorageEngine = function ExtensionStorageEngine(service) {
+  SyncEngine.call(this, "Extension-Storage", service);
+};
+ExtensionStorageEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _trackerObj: ExtensionStorageTracker,
+  // we don't need these since we implement our own sync logic
+  _storeObj: undefined,
+  _recordObj: undefined,
+
+  syncPriority: 10,
+
+  _sync: function () {
+    return Async.promiseSpinningly(ExtensionStorageSync.syncAll());
+  },
+
+  get enabled() {
+    // By default, we sync extension storage if we sync addons. This
+    // lets us simplify the UX since users probably don't consider
+    // "extension preferences" a separate category of syncing.
+    // However, we also respect engine.extension-storage.force, which
+    // can be set to true or false, if a power user wants to customize
+    // the behavior despite the lack of UI.
+    const forced = Svc.Prefs.get("engine." + this.prefName + ".force", undefined);
+    if (forced !== undefined) {
+      return forced;
+    }
+    return Svc.Prefs.get("engine.addons", false);
+  },
+};
+
+function ExtensionStorageTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+ExtensionStorageTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  startTracking: function () {
+    Svc.Obs.add("ext.storage.sync-changed", this);
+  },
+
+  stopTracking: function () {
+    Svc.Obs.remove("ext.storage.sync-changed", this);
+  },
+
+  observe: function (subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+
+    if (this.ignoreAll) {
+      return;
+    }
+
+    if (topic !== "ext.storage.sync-changed") {
+      return;
+    }
+
+    // Single adds, removes and changes are not so important on their
+    // own, so let's just increment score a bit.
+    this.score += SCORE_INCREMENT_MEDIUM;
+  },
+
+  // Override a bunch of methods which don't do anything for us.
+  // This is a performance hack.
+  saveChangedIDs: function() {
+  },
+  loadChangedIDs: function() {
+  },
+  ignoreID: function() {
+  },
+  unignoreID: function() {
+  },
+  addChangedID: function() {
+  },
+  removeChangedID: function() {
+  },
+  clearChangedIDs: function() {
+  },
+};
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -39,16 +39,17 @@ Cu.import("resource://services-sync/util
 const ENGINE_MODULES = {
   Addons: "addons.js",
   Bookmarks: "bookmarks.js",
   Form: "forms.js",
   History: "history.js",
   Password: "passwords.js",
   Prefs: "prefs.js",
   Tab: "tabs.js",
+  ExtensionStorage: "extension-storage.js",
 };
 
 const STORAGE_INFO_TYPES = [INFO_COLLECTIONS,
                             INFO_COLLECTION_USAGE,
                             INFO_COLLECTION_COUNTS,
                             INFO_QUOTA];
 
 function Sync11Service() {
--- a/services/sync/modules/telemetry.js
+++ b/services/sync/modules/telemetry.js
@@ -46,17 +46,17 @@ const TOPICS = [
   "weave:engine:validate:finish",
   "weave:engine:validate:error",
 ];
 
 const PING_FORMAT_VERSION = 1;
 
 // The set of engines we record telemetry for - any other engines are ignored.
 const ENGINES = new Set(["addons", "bookmarks", "clients", "forms", "history",
-                         "passwords", "prefs", "tabs"]);
+                         "passwords", "prefs", "tabs", "extension-storage"]);
 
 // A regex we can use to replace the profile dir in error messages. We use a
 // regexp so we can simply replace all case-insensitive occurences.
 // This escaping function is from:
 // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions
 const reProfileDir = new RegExp(
         OS.Constants.Path.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"),
         "gi");
--- a/services/sync/moz.build
+++ b/services/sync/moz.build
@@ -47,16 +47,17 @@ EXTRA_PP_JS_MODULES['services-sync'] += 
 # Definitions used by constants.js
 DEFINES['weave_version'] = '1.54.0'
 DEFINES['weave_id'] = '{340c2bbc-ce74-4362-90b5-7c26312808ef}'
 
 EXTRA_JS_MODULES['services-sync'].engines += [
     'modules/engines/addons.js',
     'modules/engines/bookmarks.js',
     'modules/engines/clients.js',
+    'modules/engines/extension-storage.js',
     'modules/engines/forms.js',
     'modules/engines/history.js',
     'modules/engines/passwords.js',
     'modules/engines/prefs.js',
     'modules/engines/tabs.js',
 ]
 
 EXTRA_JS_MODULES['services-sync'].stages += [
--- a/services/sync/services-sync.js
+++ b/services/sync/services-sync.js
@@ -26,16 +26,17 @@ pref("services.sync.errorhandler.network
 
 pref("services.sync.engine.addons", true);
 pref("services.sync.engine.bookmarks", true);
 pref("services.sync.engine.history", true);
 pref("services.sync.engine.passwords", true);
 pref("services.sync.engine.prefs", true);
 pref("services.sync.engine.tabs", true);
 pref("services.sync.engine.tabs.filteredUrls", "^(about:.*|chrome://weave/.*|wyciwyg:.*|file:.*|blob:.*)$");
+pref("services.sync.engine.extension-storage", true);
 
 pref("services.sync.jpake.serverURL", "https://setup.services.mozilla.com/");
 pref("services.sync.jpake.pollInterval", 1000);
 pref("services.sync.jpake.firstMsgMaxTries", 300); // 5 minutes
 pref("services.sync.jpake.lastMsgMaxTries", 300);  // 5 minutes
 pref("services.sync.jpake.maxTries", 10);
 
 // If true, add-on sync ignores changes to the user-enabled flag. This
@@ -63,16 +64,17 @@ pref("services.sync.log.logger.service.j
 pref("services.sync.log.logger.engine.bookmarks", "Debug");
 pref("services.sync.log.logger.engine.clients", "Debug");
 pref("services.sync.log.logger.engine.forms", "Debug");
 pref("services.sync.log.logger.engine.history", "Debug");
 pref("services.sync.log.logger.engine.passwords", "Debug");
 pref("services.sync.log.logger.engine.prefs", "Debug");
 pref("services.sync.log.logger.engine.tabs", "Debug");
 pref("services.sync.log.logger.engine.addons", "Debug");
+pref("services.sync.log.logger.engine.extension-storage", "Debug");
 pref("services.sync.log.logger.engine.apps", "Debug");
 pref("services.sync.log.logger.identity", "Debug");
 pref("services.sync.log.logger.userapi", "Debug");
 pref("services.sync.log.cryptoDebug", false);
 
 pref("services.sync.fxa.termsURL", "https://accounts.firefox.com/legal/terms");
 pref("services.sync.fxa.privacyURL", "https://accounts.firefox.com/legal/privacy");
 
--- a/services/sync/tests/unit/head_helpers.js
+++ b/services/sync/tests/unit/head_helpers.js
@@ -71,16 +71,34 @@ function ExtensionsTestPath(path) {
 function loadAddonTestFunctions() {
   const path = ExtensionsTestPath("/head_addons.js");
   let file = do_get_file(path);
   let uri = Services.io.newFileURI(file);
   Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
 }
 
+function webExtensionsTestPath(path) {
+  if (path[0] != "/") {
+    throw Error("Path must begin with '/': " + path);
+  }
+
+  return "../../../../toolkit/components/extensions/test/xpcshell" + path;
+}
+
+/**
+ * Loads the WebExtension test functions by importing its test file.
+ */
+function loadWebExtensionTestFunctions() {
+  const path = webExtensionsTestPath("/head_sync.js");
+  let file = do_get_file(path);
+  let uri = Services.io.newFileURI(file);
+  Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
+}
+
 function getAddonInstall(name) {
   let f = do_get_file(ExtensionsTestPath("/addons/" + name + ".xpi"));
   let cb = Async.makeSyncCallback();
   AddonManager.getInstallForFile(f, cb);
 
   return Async.waitForSyncCallback(cb);
 }
 
new file mode 100644
--- /dev/null
+++ b/services/sync/tests/unit/test_extension_storage_engine.js
@@ -0,0 +1,62 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/extension-storage.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://testing-common/services/sync/utils.js");
+Cu.import("resource://gre/modules/ExtensionStorageSync.jsm");
+
+Service.engineManager.register(ExtensionStorageEngine);
+const engine = Service.engineManager.get("extension-storage");
+do_get_profile();   // so we can use FxAccounts
+loadWebExtensionTestFunctions();
+
+function mock(options) {
+  let calls = [];
+  let ret = function() {
+    calls.push(arguments);
+    return options.returns;
+  }
+  Object.setPrototypeOf(ret, {
+    __proto__: Function.prototype,
+    get calls() {
+      return calls;
+    }
+  });
+  return ret;
+}
+
+add_task(function* test_calling_sync_calls__sync() {
+  let oldSync = ExtensionStorageEngine.prototype._sync;
+  let syncMock = ExtensionStorageEngine.prototype._sync = mock({returns: true});
+  try {
+    // I wanted to call the main sync entry point for the entire
+    // package, but that fails because it tries to sync ClientEngine
+    // first, which fails.
+    yield engine.sync();
+  } finally {
+    ExtensionStorageEngine.prototype._sync = oldSync;
+  }
+  equal(syncMock.calls.length, 1);
+});
+
+add_task(function* test_calling_sync_calls_ext_storage_sync() {
+  const extension = {id: "my-extension"};
+  let oldSync = ExtensionStorageSync.syncAll;
+  let syncMock = ExtensionStorageSync.syncAll = mock({returns: Promise.resolve()});
+  try {
+    yield* withSyncContext(function* (context) {
+      // Set something so that everyone knows that we're using storage.sync
+      yield ExtensionStorageSync.set(extension, {"a": "b"}, context);
+
+      yield engine._sync();
+    });
+  } finally {
+    ExtensionStorageSync.syncAll = oldSync;
+  }
+  do_check_true(syncMock.calls.length >= 1);
+});
new file mode 100644
--- /dev/null
+++ b/services/sync/tests/unit/test_extension_storage_tracker.js
@@ -0,0 +1,38 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/extension-storage.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://gre/modules/ExtensionStorageSync.jsm");
+
+Service.engineManager.register(ExtensionStorageEngine);
+const engine = Service.engineManager.get("extension-storage");
+do_get_profile();   // so we can use FxAccounts
+loadWebExtensionTestFunctions();
+
+add_task(function* test_changing_extension_storage_changes_score() {
+  const tracker = engine._tracker;
+  const extension = {id: "my-extension-id"};
+  Svc.Obs.notify("weave:engine:start-tracking");
+  yield* withSyncContext(function*(context) {
+    yield ExtensionStorageSync.set(extension, {"a": "b"}, context);
+  });
+  do_check_eq(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  tracker.resetScore();
+  yield* withSyncContext(function*(context) {
+    yield ExtensionStorageSync.remove(extension, "a", context);
+  });
+  do_check_eq(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  Svc.Obs.notify("weave:engine:stop-tracking");
+});
+
+function run_test() {
+  run_next_test();
+}
--- a/services/sync/tests/unit/test_load_modules.js
+++ b/services/sync/tests/unit/test_load_modules.js
@@ -4,16 +4,17 @@
 const modules = [
   "addonutils.js",
   "addonsreconciler.js",
   "browserid_identity.js",
   "constants.js",
   "engines/addons.js",
   "engines/bookmarks.js",
   "engines/clients.js",
+  "engines/extension-storage.js",
   "engines/forms.js",
   "engines/history.js",
   "engines/passwords.js",
   "engines/prefs.js",
   "engines/tabs.js",
   "engines.js",
   "identity.js",
   "jpakeclient.js",
--- a/services/sync/tests/unit/xpcshell.ini
+++ b/services/sync/tests/unit/xpcshell.ini
@@ -9,16 +9,17 @@ support-files =
   missing-sourceuri.xml
   missing-xpi-search.xml
   places_v10_from_v11.sqlite
   rewrite-search.xml
   sync_ping_schema.json
   systemaddon-search.xml
   !/services/common/tests/unit/head_helpers.js
   !/toolkit/mozapps/extensions/test/xpcshell/head_addons.js
+  !/toolkit/components/extensions/test/xpcshell/head_sync.js
 
 # The manifest is roughly ordered from low-level to high-level. When making
 # systemic sweeping changes, this makes it easier to identify errors closer to
 # the source.
 
 # Ensure we can import everything.
 [test_load_modules.js]
 
@@ -156,16 +157,18 @@ tags = addons
 [test_bookmark_smart_bookmarks.js]
 [test_bookmark_store.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)
 skip-if = debug
 [test_bookmark_tracker.js]
 [test_bookmark_validator.js]
 [test_clients_engine.js]
 [test_clients_escape.js]
+[test_extension_storage_engine.js]
+[test_extension_storage_tracker.js]
 [test_forms_store.js]
 [test_forms_tracker.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)
 skip-if = debug
 [test_history_engine.js]
 [test_history_store.js]
 [test_history_tracker.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)
--- a/toolkit/components/extensions/ExtensionStorageSync.jsm
+++ b/toolkit/components/extensions/ExtensionStorageSync.jsm
@@ -22,16 +22,18 @@ const {
 XPCOMUtils.defineLazyModuleGetter(this, "AppsUtils",
                                   "resource://gre/modules/AppsUtils.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "AsyncShutdown",
                                   "resource://gre/modules/AsyncShutdown.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorage",
                                   "resource://gre/modules/ExtensionStorage.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "loadKinto",
                                   "resource://services-common/kinto-offline-client.js");
+XPCOMUtils.defineLazyModuleGetter(this, "Observers",
+                                  "resource://services-common/observers.js");
 XPCOMUtils.defineLazyModuleGetter(this, "Sqlite",
                                   "resource://gre/modules/Sqlite.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "Task",
                                   "resource://gre/modules/Task.jsm");
 XPCOMUtils.defineLazyPreferenceGetter(this, "prefPermitsStorageSync",
                                       STORAGE_SYNC_ENABLED_PREF, false);
 
 /* globals prefPermitsStorageSync */
@@ -311,16 +313,17 @@ this.ExtensionStorageSync = {
     let listeners = this.listeners.get(extension);
     listeners.delete(listener);
     if (listeners.size == 0) {
       this.listeners.delete(extension);
     }
   },
 
   notifyListeners(extension, changes) {
+    Observers.notify("ext.storage.sync-changed");
     let listeners = this.listeners.get(extension) || new Set();
     if (listeners) {
       for (let listener of listeners) {
         runSafeSyncWithoutClone(listener, changes);
       }
     }
   },
 };
new file mode 100644
--- /dev/null
+++ b/toolkit/components/extensions/test/xpcshell/head_sync.js
@@ -0,0 +1,67 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/* exported withSyncContext */
+
+Components.utils.import("resource://gre/modules/Services.jsm", this);
+Components.utils.import("resource://gre/modules/ExtensionCommon.jsm", this);
+
+var {
+  BaseContext,
+} = ExtensionCommon;
+
+class Context extends BaseContext {
+  constructor(principal) {
+    super();
+    Object.defineProperty(this, "principal", {
+      value: principal,
+      configurable: true,
+    });
+    this.sandbox = Components.utils.Sandbox(principal, {wantXrays: false});
+    this.extension = {id: "test@web.extension"};
+  }
+
+  get cloneScope() {
+    return this.sandbox;
+  }
+}
+
+/**
+ * Call the given function with a newly-constructed context.
+ * Unload the context on the way out.
+ *
+ * @param {function} f    the function to call
+ */
+function* withContext(f) {
+  const ssm = Services.scriptSecurityManager;
+  const PRINCIPAL1 = ssm.createCodebasePrincipalFromOrigin("http://www.example.org");
+  const context = new Context(PRINCIPAL1);
+  try {
+    yield* f(context);
+  } finally {
+    yield context.unload();
+  }
+}
+
+/**
+ * Like withContext(), but also turn on the "storage.sync" pref for
+ * the duration of the function.
+ * Calls to this function can be replaced with calls to withContext
+ * once the pref becomes on by default.
+ *
+ * @param {function} f    the function to call
+ */
+function* withSyncContext(f) {
+  const STORAGE_SYNC_PREF = "webextensions.storage.sync.enabled";
+  let prefs = Services.prefs;
+
+  try {
+    prefs.setBoolPref(STORAGE_SYNC_PREF, true);
+    yield* withContext(f);
+  } finally {
+    prefs.clearUserPref(STORAGE_SYNC_PREF);
+  }
+}
--- a/toolkit/components/extensions/test/xpcshell/xpcshell.ini
+++ b/toolkit/components/extensions/test/xpcshell/xpcshell.ini
@@ -1,15 +1,15 @@
 [DEFAULT]
 head = head.js
 tail =
 firefox-appdir = browser
 skip-if = appname == "thunderbird"
 support-files =
-  data/**
+  data/** head_sync.js
 tags = webextensions
 
 [test_csp_custom_policies.js]
 [test_csp_validator.js]
 [test_ext_alarms.js]
 [test_ext_alarms_does_not_fire.js]
 [test_ext_alarms_periodic.js]
 [test_ext_alarms_replaces.js]