Bug 1253740 - Introduce extension-storage engine with a sanity test, r=markh
☠☠ backed out by 35e7f8176332 ☠ ☠
authorEthan Glasser-Camp <eglassercamp@mozilla.com>
Thu, 08 Sep 2016 14:23:12 -0400
changeset 322228 653788e146f2f41b388297b9ad66eb493db987c4
parent 322227 b98fa03d8c155826643526d7a6f94ecc089a5cbf
child 322229 4fbff333994f553d4416cfdc285fa43a6268c842
push id21
push usermaklebus@msu.edu
push dateThu, 01 Dec 2016 06:22:08 +0000
reviewersmarkh
bugs1253740
milestone52.0a1
Bug 1253740 - Introduce extension-storage engine with a sanity test, r=markh Note that this "enables" the engine using a pref, even though it might not be ready yet, so that the tests can pass. MozReview-Commit-ID: AZ0TVERiQDU
browser/app/profile/firefox.js
services/sync/modules/engines/extension-storage.js
services/sync/modules/service.js
services/sync/modules/telemetry.js
services/sync/moz.build
services/sync/services-sync.js
services/sync/tests/unit/head_helpers.js
services/sync/tests/unit/test_extension_storage_engine.js
services/sync/tests/unit/test_extension_storage_tracker.js
services/sync/tests/unit/test_load_modules.js
services/sync/tests/unit/xpcshell.ini
toolkit/components/extensions/ExtensionStorageSync.jsm
toolkit/components/extensions/test/xpcshell/head_sync.js
toolkit/components/extensions/test/xpcshell/xpcshell.ini
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1038,17 +1038,17 @@ pref("browser.taskbar.lists.enabled", tr
 pref("browser.taskbar.lists.frequent.enabled", true);
 pref("browser.taskbar.lists.recent.enabled", false);
 pref("browser.taskbar.lists.maxListItemCount", 7);
 pref("browser.taskbar.lists.tasks.enabled", true);
 pref("browser.taskbar.lists.refreshInSeconds", 120);
 #endif
 
 // The sync engines to use.
-pref("services.sync.registerEngines", "Bookmarks,Form,History,Password,Prefs,Tab,Addons");
+pref("services.sync.registerEngines", "Bookmarks,Form,History,Password,Prefs,Tab,Addons,ExtensionStorage");
 // Preferences to be synced by default
 pref("services.sync.prefs.sync.accessibility.blockautorefresh", true);
 pref("services.sync.prefs.sync.accessibility.browsewithcaret", true);
 pref("services.sync.prefs.sync.accessibility.typeaheadfind", true);
 pref("services.sync.prefs.sync.accessibility.typeaheadfind.linksonly", true);
 pref("services.sync.prefs.sync.addons.ignoreUserEnabledChanges", true);
 // The addons prefs related to repository verification are intentionally
 // not synced for security reasons. If a system is compromised, a user
new file mode 100644
--- /dev/null
+++ b/services/sync/modules/engines/extension-storage.js
@@ -0,0 +1,103 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ['ExtensionStorageEngine'];
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorageSync",
+                                  "resource://gre/modules/ExtensionStorageSync.jsm");
+
+/**
+ * The Engine that manages syncing for the web extension "storage"
+ * API, and in particular ext.storage.sync.
+ *
+ * ext.storage.sync is implemented using Kinto, so it has mechanisms
+ * for syncing that we do not need to integrate in the Firefox Sync
+ * framework, so this is something of a stub.
+ */
+this.ExtensionStorageEngine = function ExtensionStorageEngine(service) {
+  SyncEngine.call(this, "Extension-Storage", service);
+};
+ExtensionStorageEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _trackerObj: ExtensionStorageTracker,
+  // we don't need these since we implement our own sync logic
+  _storeObj: undefined,
+  _recordObj: undefined,
+
+  syncPriority: 10,
+
+  _sync: function () {
+    return Async.promiseSpinningly(ExtensionStorageSync.syncAll());
+  },
+
+  get enabled() {
+    // By default, we sync extension storage if we sync addons. This
+    // lets us simplify the UX since users probably don't consider
+    // "extension preferences" a separate category of syncing.
+    // However, we also respect engine.extension-storage.force, which
+    // can be set to true or false, if a power user wants to customize
+    // the behavior despite the lack of UI.
+    const forced = Svc.Prefs.get("engine." + this.prefName + ".force", undefined);
+    if (forced !== undefined) {
+      return forced;
+    }
+    return Svc.Prefs.get("engine.addons", false);
+  },
+};
+
+function ExtensionStorageTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+ExtensionStorageTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  startTracking: function () {
+    Svc.Obs.add("ext.storage.sync-changed", this);
+  },
+
+  stopTracking: function () {
+    Svc.Obs.remove("ext.storage.sync-changed", this);
+  },
+
+  observe: function (subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+
+    if (this.ignoreAll) {
+      return;
+    }
+
+    if (topic !== "ext.storage.sync-changed") {
+      return;
+    }
+
+    // Single adds, removes and changes are not so important on their
+    // own, so let's just increment score a bit.
+    this.score += SCORE_INCREMENT_MEDIUM;
+  },
+
+  // Override a bunch of methods which don't do anything for us.
+  // This is a performance hack.
+  saveChangedIDs: function() {
+  },
+  loadChangedIDs: function() {
+  },
+  ignoreID: function() {
+  },
+  unignoreID: function() {
+  },
+  addChangedID: function() {
+  },
+  removeChangedID: function() {
+  },
+  clearChangedIDs: function() {
+  },
+};
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -39,16 +39,17 @@ Cu.import("resource://services-sync/util
 const ENGINE_MODULES = {
   Addons: "addons.js",
   Bookmarks: "bookmarks.js",
   Form: "forms.js",
   History: "history.js",
   Password: "passwords.js",
   Prefs: "prefs.js",
   Tab: "tabs.js",
+  ExtensionStorage: "extension-storage.js",
 };
 
 const STORAGE_INFO_TYPES = [INFO_COLLECTIONS,
                             INFO_COLLECTION_USAGE,
                             INFO_COLLECTION_COUNTS,
                             INFO_QUOTA];
 
 function Sync11Service() {
--- a/services/sync/modules/telemetry.js
+++ b/services/sync/modules/telemetry.js
@@ -46,17 +46,17 @@ const TOPICS = [
   "weave:engine:validate:finish",
   "weave:engine:validate:error",
 ];
 
 const PING_FORMAT_VERSION = 1;
 
 // The set of engines we record telemetry for - any other engines are ignored.
 const ENGINES = new Set(["addons", "bookmarks", "clients", "forms", "history",
-                         "passwords", "prefs", "tabs"]);
+                         "passwords", "prefs", "tabs", "extension-storage"]);
 
 // A regex we can use to replace the profile dir in error messages. We use a
 // regexp so we can simply replace all case-insensitive occurences.
 // This escaping function is from:
 // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions
 const reProfileDir = new RegExp(
         OS.Constants.Path.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"),
         "gi");
--- a/services/sync/moz.build
+++ b/services/sync/moz.build
@@ -47,16 +47,17 @@ EXTRA_PP_JS_MODULES['services-sync'] += 
 # Definitions used by constants.js
 DEFINES['weave_version'] = '1.54.0'
 DEFINES['weave_id'] = '{340c2bbc-ce74-4362-90b5-7c26312808ef}'
 
 EXTRA_JS_MODULES['services-sync'].engines += [
     'modules/engines/addons.js',
     'modules/engines/bookmarks.js',
     'modules/engines/clients.js',
+    'modules/engines/extension-storage.js',
     'modules/engines/forms.js',
     'modules/engines/history.js',
     'modules/engines/passwords.js',
     'modules/engines/prefs.js',
     'modules/engines/tabs.js',
 ]
 
 EXTRA_JS_MODULES['services-sync'].stages += [
--- a/services/sync/services-sync.js
+++ b/services/sync/services-sync.js
@@ -26,16 +26,17 @@ pref("services.sync.errorhandler.network
 
 pref("services.sync.engine.addons", true);
 pref("services.sync.engine.bookmarks", true);
 pref("services.sync.engine.history", true);
 pref("services.sync.engine.passwords", true);
 pref("services.sync.engine.prefs", true);
 pref("services.sync.engine.tabs", true);
 pref("services.sync.engine.tabs.filteredUrls", "^(about:.*|chrome://weave/.*|wyciwyg:.*|file:.*|blob:.*)$");
+pref("services.sync.engine.extension-storage", true);
 
 pref("services.sync.jpake.serverURL", "https://setup.services.mozilla.com/");
 pref("services.sync.jpake.pollInterval", 1000);
 pref("services.sync.jpake.firstMsgMaxTries", 300); // 5 minutes
 pref("services.sync.jpake.lastMsgMaxTries", 300);  // 5 minutes
 pref("services.sync.jpake.maxTries", 10);
 
 // If true, add-on sync ignores changes to the user-enabled flag. This
@@ -63,16 +64,17 @@ pref("services.sync.log.logger.service.j
 pref("services.sync.log.logger.engine.bookmarks", "Debug");
 pref("services.sync.log.logger.engine.clients", "Debug");
 pref("services.sync.log.logger.engine.forms", "Debug");
 pref("services.sync.log.logger.engine.history", "Debug");
 pref("services.sync.log.logger.engine.passwords", "Debug");
 pref("services.sync.log.logger.engine.prefs", "Debug");
 pref("services.sync.log.logger.engine.tabs", "Debug");
 pref("services.sync.log.logger.engine.addons", "Debug");
+pref("services.sync.log.logger.engine.extension-storage", "Debug");
 pref("services.sync.log.logger.engine.apps", "Debug");
 pref("services.sync.log.logger.identity", "Debug");
 pref("services.sync.log.logger.userapi", "Debug");
 pref("services.sync.log.cryptoDebug", false);
 
 pref("services.sync.fxa.termsURL", "https://accounts.firefox.com/legal/terms");
 pref("services.sync.fxa.privacyURL", "https://accounts.firefox.com/legal/privacy");
 
--- a/services/sync/tests/unit/head_helpers.js
+++ b/services/sync/tests/unit/head_helpers.js
@@ -71,16 +71,34 @@ function ExtensionsTestPath(path) {
 function loadAddonTestFunctions() {
   const path = ExtensionsTestPath("/head_addons.js");
   let file = do_get_file(path);
   let uri = Services.io.newFileURI(file);
   Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
 }
 
+function webExtensionsTestPath(path) {
+  if (path[0] != "/") {
+    throw Error("Path must begin with '/': " + path);
+  }
+
+  return "../../../../toolkit/components/extensions/test/xpcshell" + path;
+}
+
+/**
+ * Loads the WebExtension test functions by importing its test file.
+ */
+function loadWebExtensionTestFunctions() {
+  const path = webExtensionsTestPath("/head_sync.js");
+  let file = do_get_file(path);
+  let uri = Services.io.newFileURI(file);
+  Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
+}
+
 function getAddonInstall(name) {
   let f = do_get_file(ExtensionsTestPath("/addons/" + name + ".xpi"));
   let cb = Async.makeSyncCallback();
   AddonManager.getInstallForFile(f, cb);
 
   return Async.waitForSyncCallback(cb);
 }
 
new file mode 100644
--- /dev/null
+++ b/services/sync/tests/unit/test_extension_storage_engine.js
@@ -0,0 +1,62 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/extension-storage.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://testing-common/services/sync/utils.js");
+Cu.import("resource://gre/modules/ExtensionStorageSync.jsm");
+
+Service.engineManager.register(ExtensionStorageEngine);
+const engine = Service.engineManager.get("extension-storage");
+do_get_profile();   // so we can use FxAccounts
+loadWebExtensionTestFunctions();
+
+function mock(options) {
+  let calls = [];
+  let ret = function() {
+    calls.push(arguments);
+    return options.returns;
+  }
+  Object.setPrototypeOf(ret, {
+    __proto__: Function.prototype,
+    get calls() {
+      return calls;
+    }
+  });
+  return ret;
+}
+
+add_task(function* test_calling_sync_calls__sync() {
+  let oldSync = ExtensionStorageEngine.prototype._sync;
+  let syncMock = ExtensionStorageEngine.prototype._sync = mock({returns: true});
+  try {
+    // I wanted to call the main sync entry point for the entire
+    // package, but that fails because it tries to sync ClientEngine
+    // first, which fails.
+    yield engine.sync();
+  } finally {
+    ExtensionStorageEngine.prototype._sync = oldSync;
+  }
+  equal(syncMock.calls.length, 1);
+});
+
+add_task(function* test_calling_sync_calls_ext_storage_sync() {
+  const extension = {id: "my-extension"};
+  let oldSync = ExtensionStorageSync.syncAll;
+  let syncMock = ExtensionStorageSync.syncAll = mock({returns: Promise.resolve()});
+  try {
+    yield* withSyncContext(function* (context) {
+      // Set something so that everyone knows that we're using storage.sync
+      yield ExtensionStorageSync.set(extension, {"a": "b"}, context);
+
+      yield engine._sync();
+    });
+  } finally {
+    ExtensionStorageSync.syncAll = oldSync;
+  }
+  do_check_true(syncMock.calls.length >= 1);
+});
new file mode 100644
--- /dev/null
+++ b/services/sync/tests/unit/test_extension_storage_tracker.js
@@ -0,0 +1,38 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/extension-storage.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://gre/modules/ExtensionStorageSync.jsm");
+
+Service.engineManager.register(ExtensionStorageEngine);
+const engine = Service.engineManager.get("extension-storage");
+do_get_profile();   // so we can use FxAccounts
+loadWebExtensionTestFunctions();
+
+add_task(function* test_changing_extension_storage_changes_score() {
+  const tracker = engine._tracker;
+  const extension = {id: "my-extension-id"};
+  Svc.Obs.notify("weave:engine:start-tracking");
+  yield* withSyncContext(function*(context) {
+    yield ExtensionStorageSync.set(extension, {"a": "b"}, context);
+  });
+  do_check_eq(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  tracker.resetScore();
+  yield* withSyncContext(function*(context) {
+    yield ExtensionStorageSync.remove(extension, "a", context);
+  });
+  do_check_eq(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  Svc.Obs.notify("weave:engine:stop-tracking");
+});
+
+function run_test() {
+  run_next_test();
+}
--- a/services/sync/tests/unit/test_load_modules.js
+++ b/services/sync/tests/unit/test_load_modules.js
@@ -4,16 +4,17 @@
 const modules = [
   "addonutils.js",
   "addonsreconciler.js",
   "browserid_identity.js",
   "constants.js",
   "engines/addons.js",
   "engines/bookmarks.js",
   "engines/clients.js",
+  "engines/extension-storage.js",
   "engines/forms.js",
   "engines/history.js",
   "engines/passwords.js",
   "engines/prefs.js",
   "engines/tabs.js",
   "engines.js",
   "identity.js",
   "jpakeclient.js",
--- a/services/sync/tests/unit/xpcshell.ini
+++ b/services/sync/tests/unit/xpcshell.ini
@@ -9,16 +9,17 @@ support-files =
   missing-sourceuri.xml
   missing-xpi-search.xml
   places_v10_from_v11.sqlite
   rewrite-search.xml
   sync_ping_schema.json
   systemaddon-search.xml
   !/services/common/tests/unit/head_helpers.js
   !/toolkit/mozapps/extensions/test/xpcshell/head_addons.js
+  !/toolkit/components/extensions/test/xpcshell/head_sync.js
 
 # The manifest is roughly ordered from low-level to high-level. When making
 # systemic sweeping changes, this makes it easier to identify errors closer to
 # the source.
 
 # Ensure we can import everything.
 [test_load_modules.js]
 
@@ -156,16 +157,18 @@ tags = addons
 [test_bookmark_smart_bookmarks.js]
 [test_bookmark_store.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)
 skip-if = debug
 [test_bookmark_tracker.js]
 [test_bookmark_validator.js]
 [test_clients_engine.js]
 [test_clients_escape.js]
+[test_extension_storage_engine.js]
+[test_extension_storage_tracker.js]
 [test_forms_store.js]
 [test_forms_tracker.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)
 skip-if = debug
 [test_history_engine.js]
 [test_history_store.js]
 [test_history_tracker.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)
--- a/toolkit/components/extensions/ExtensionStorageSync.jsm
+++ b/toolkit/components/extensions/ExtensionStorageSync.jsm
@@ -22,16 +22,18 @@ const {
 XPCOMUtils.defineLazyModuleGetter(this, "AppsUtils",
                                   "resource://gre/modules/AppsUtils.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "AsyncShutdown",
                                   "resource://gre/modules/AsyncShutdown.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorage",
                                   "resource://gre/modules/ExtensionStorage.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "loadKinto",
                                   "resource://services-common/kinto-offline-client.js");
+XPCOMUtils.defineLazyModuleGetter(this, "Observers",
+                                  "resource://services-common/observers.js");
 XPCOMUtils.defineLazyModuleGetter(this, "Sqlite",
                                   "resource://gre/modules/Sqlite.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "Task",
                                   "resource://gre/modules/Task.jsm");
 XPCOMUtils.defineLazyPreferenceGetter(this, "prefPermitsStorageSync",
                                       STORAGE_SYNC_ENABLED_PREF, false);
 
 /* globals prefPermitsStorageSync */
@@ -311,16 +313,17 @@ this.ExtensionStorageSync = {
     let listeners = this.listeners.get(extension);
     listeners.delete(listener);
     if (listeners.size == 0) {
       this.listeners.delete(extension);
     }
   },
 
   notifyListeners(extension, changes) {
+    Observers.notify("ext.storage.sync-changed");
     let listeners = this.listeners.get(extension) || new Set();
     if (listeners) {
       for (let listener of listeners) {
         runSafeSyncWithoutClone(listener, changes);
       }
     }
   },
 };
new file mode 100644
--- /dev/null
+++ b/toolkit/components/extensions/test/xpcshell/head_sync.js
@@ -0,0 +1,67 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/* exported withSyncContext */
+
+Components.utils.import("resource://gre/modules/Services.jsm", this);
+Components.utils.import("resource://gre/modules/ExtensionUtils.jsm", this);
+
+var {
+  BaseContext,
+} = ExtensionUtils;
+
+class Context extends BaseContext {
+  constructor(principal) {
+    super();
+    Object.defineProperty(this, "principal", {
+      value: principal,
+      configurable: true,
+    });
+    this.sandbox = Components.utils.Sandbox(principal, {wantXrays: false});
+    this.extension = {id: "test@web.extension"};
+  }
+
+  get cloneScope() {
+    return this.sandbox;
+  }
+}
+
+/**
+ * Call the given function with a newly-constructed context.
+ * Unload the context on the way out.
+ *
+ * @param {function} f    the function to call
+ */
+function* withContext(f) {
+  const ssm = Services.scriptSecurityManager;
+  const PRINCIPAL1 = ssm.createCodebasePrincipalFromOrigin("http://www.example.org");
+  const context = new Context(PRINCIPAL1);
+  try {
+    yield* f(context);
+  } finally {
+    yield context.unload();
+  }
+}
+
+/**
+ * Like withContext(), but also turn on the "storage.sync" pref for
+ * the duration of the function.
+ * Calls to this function can be replaced with calls to withContext
+ * once the pref becomes on by default.
+ *
+ * @param {function} f    the function to call
+ */
+function* withSyncContext(f) {
+  const STORAGE_SYNC_PREF = "webextensions.storage.sync.enabled";
+  let prefs = Services.prefs;
+
+  try {
+    prefs.setBoolPref(STORAGE_SYNC_PREF, true);
+    yield* withContext(f);
+  } finally {
+    prefs.clearUserPref(STORAGE_SYNC_PREF);
+  }
+}
--- a/toolkit/components/extensions/test/xpcshell/xpcshell.ini
+++ b/toolkit/components/extensions/test/xpcshell/xpcshell.ini
@@ -1,15 +1,15 @@
 [DEFAULT]
 head = head.js
 tail =
 firefox-appdir = browser
 skip-if = appname == "thunderbird"
 support-files =
-  data/**
+  data/** head_sync.js
 tags = webextensions
 
 [test_csp_custom_policies.js]
 [test_csp_validator.js]
 [test_ext_alarms.js]
 [test_ext_alarms_does_not_fire.js]
 [test_ext_alarms_periodic.js]
 [test_ext_alarms_replaces.js]