Bug 1489531 Expose telemetry client_id hash to about:addons via cookie r=Gijs,chutten
authorShane Caraveo <scaraveo@mozilla.com>
Mon, 26 Nov 2018 15:26:39 +0000
changeset 504427 374a4bf1482e462845215fded70e4f7854b10271
parent 504426 ebe007c97c5dd1a6d9153f2b8cea84e27601470e
child 504428 ef8fb4300e67a3857a65a2c9aa2528f65c6a07cb
push id10290
push userffxbld-merge
push dateMon, 03 Dec 2018 16:23:23 +0000
treeherdermozilla-beta@700bed2445e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersGijs, chutten
bugs1489531
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1489531 Expose telemetry client_id hash to about:addons via cookie r=Gijs,chutten Differential Revision: https://phabricator.services.mozilla.com/D9317
browser/app/profile/firefox.js
browser/components/nsBrowserGlue.js
browser/modules/Discovery.jsm
browser/modules/moz.build
browser/modules/test/unit/test_discovery.js
browser/modules/test/unit/xpcshell.ini
toolkit/modules/ClientID.jsm
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1789,8 +1789,13 @@ pref("prio.publicKeyB", "26E6674E65425B8
 
 // Coverage ping is disabled by default.
 pref("toolkit.coverage.enabled", false);
 pref("toolkit.coverage.endpoint.base", "https://coverage.mozilla.org");
 // Whether or not Prio-encoded Telemetry will be sent along with the main ping.
 #if defined(NIGHTLY_BUILD) && defined(MOZ_LIBPRIO)
 pref("prio.enabled", true);
 #endif
+
+// Discovery prefs
+pref("browser.discovery.enabled", false);
+pref("browser.discovery.containers.enabled", true);
+pref("browser.discovery.sites", "addons.mozilla.org");
--- a/browser/components/nsBrowserGlue.js
+++ b/browser/components/nsBrowserGlue.js
@@ -386,16 +386,17 @@ XPCOMUtils.defineLazyModuleGetters(this,
   BookmarkJSONUtils: "resource://gre/modules/BookmarkJSONUtils.jsm",
   BrowserErrorReporter: "resource:///modules/BrowserErrorReporter.jsm",
   BrowserUsageTelemetry: "resource:///modules/BrowserUsageTelemetry.jsm",
   BrowserWindowTracker: "resource:///modules/BrowserWindowTracker.jsm",
   ContentClick: "resource:///modules/ContentClick.jsm",
   ContextualIdentityService: "resource://gre/modules/ContextualIdentityService.jsm",
   CustomizableUI: "resource:///modules/CustomizableUI.jsm",
   DateTimePickerParent: "resource://gre/modules/DateTimePickerParent.jsm",
+  Discovery: "resource:///modules/Discovery.jsm",
   ExtensionsUI: "resource:///modules/ExtensionsUI.jsm",
   Feeds: "resource:///modules/Feeds.jsm",
   FileSource: "resource://gre/modules/L10nRegistry.jsm",
   FormValidationHandler: "resource:///modules/FormValidationHandler.jsm",
   FxAccounts: "resource://gre/modules/FxAccounts.jsm",
   HomePage: "resource:///modules/HomePage.jsm",
   HybridContentTelemetry: "resource://gre/modules/HybridContentTelemetry.jsm",
   Integration: "resource://gre/modules/Integration.jsm",
@@ -1666,16 +1667,20 @@ BrowserGlue.prototype = {
       Blocklist.loadBlocklistAsync();
     });
 
     if (Services.prefs.getIntPref("browser.livebookmarks.migrationAttemptsLeft", 0) > 0) {
       Services.tm.idleDispatchToMainThread(() => {
         LiveBookmarkMigrator.migrate().catch(Cu.reportError);
       });
     }
+
+    Services.tm.idleDispatchToMainThread(() => {
+      Discovery.update();
+    });
   },
 
   /**
    * Use this function as an entry point to schedule tasks that need
    * to run once per session, at any arbitrary point in time.
    * This function will be called from an idle observer. Check the value of
    * LATE_TASKS_IDLE_TIME_SEC to see the current value for this idle
    * observer.
new file mode 100644
--- /dev/null
+++ b/browser/modules/Discovery.jsm
@@ -0,0 +1,119 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+ "use strict";
+
+var EXPORTED_SYMBOLS = [
+  "Discovery",
+];
+
+ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
+ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+ChromeUtils.defineModuleGetter(this, "ClientID",
+                               "resource://gre/modules/ClientID.jsm");
+ChromeUtils.defineModuleGetter(this, "ContextualIdentityService",
+                               "resource://gre/modules/ContextualIdentityService.jsm");
+
+const RECOMMENDATION_ENABLED = "browser.discovery.enabled";
+const TELEMETRY_ENABLED = "datareporting.healthreport.uploadEnabled";
+const TAAR_COOKIE_NAME = "taarId";
+
+const Discovery = {
+  set enabled(val) {
+    val = !!val;
+    if (val && !gTelemetryEnabled) {
+      throw Error("unable to turn on recommendations");
+    }
+    Services.prefs.setBoolPref(RECOMMENDATION_ENABLED, val);
+  },
+
+  get enabled() {
+    return gTelemetryEnabled && gRecommendationEnabled;
+  },
+
+  reset() {
+    return DiscoveryInternal.update(true);
+  },
+
+  update() {
+    return DiscoveryInternal.update();
+  },
+};
+
+XPCOMUtils.defineLazyPreferenceGetter(this, "gRecommendationEnabled",
+                                      RECOMMENDATION_ENABLED, false,
+                                      Discovery.update);
+XPCOMUtils.defineLazyPreferenceGetter(this, "gTelemetryEnabled",
+                                      TELEMETRY_ENABLED, false,
+                                      Discovery.update);
+XPCOMUtils.defineLazyPreferenceGetter(this, "gCachedClientID",
+                                      "toolkit.telemetry.cachedClientID", "",
+                                      Discovery.reset);
+XPCOMUtils.defineLazyPreferenceGetter(this, "gContainersEnabled",
+                                      "browser.discovery.containers.enabled", false,
+                                      Discovery.reset);
+
+Services.obs.addObserver(Discovery.update, "contextual-identity-created");
+
+const DiscoveryInternal = {
+  get sites() {
+    delete this.sites;
+    this.sites = Services.prefs.getCharPref("browser.discovery.sites", "").split(",");
+    return this.sites;
+  },
+
+  getContextualIDs() {
+    // There is never a zero id, this is just for use in update.
+    let IDs = [0];
+    if (gContainersEnabled) {
+      ContextualIdentityService.getPublicIdentities().forEach(identity => {
+        IDs.push(identity.userContextId);
+      });
+    }
+    return IDs;
+  },
+
+  async update(reset = false) {
+    if (reset || !Discovery.enabled) {
+      for (let site of this.sites) {
+        Services.cookies.remove(site, TAAR_COOKIE_NAME, "/", false, {});
+        ContextualIdentityService.getPublicIdentities().forEach(identity => {
+          let {userContextId} = identity;
+          Services.cookies.remove(site, TAAR_COOKIE_NAME, "/", false, {userContextId});
+        });
+      }
+    }
+
+    if (Discovery.enabled) {
+      // If the client id is not cached, wait for the notification that it is
+      // cached.  This will happen shortly after startup in TelemetryController.jsm.
+      // When that happens, we'll get a pref notification for the cached id,
+      // which will call update again.
+      if (!gCachedClientID) {
+        return;
+      }
+      let id = await ClientID.getClientIdHash();
+      for (let site of this.sites) {
+        // This cookie gets tied down as much as possible.  Specifically,
+        // SameSite, Secure, HttpOnly and non-PrivateBrowsing.
+        for (let userContextId of this.getContextualIDs()) {
+          let originAttributes = {privateBrowsingId: 0};
+          if (userContextId > 0) {
+            originAttributes.userContextId = userContextId;
+          }
+          if (Services.cookies.cookieExists(site, "/", TAAR_COOKIE_NAME, originAttributes)) {
+            continue;
+          }
+          Services.cookies.add(site, "/", TAAR_COOKIE_NAME, id,
+                              true, // secure
+                              true, // httpOnly
+                              true, // session
+                              Date.now(),
+                              originAttributes,
+                              Ci.nsICookie2.SAMESITE_LAX);
+        }
+      }
+    }
+  },
+};
--- a/browser/modules/moz.build
+++ b/browser/modules/moz.build
@@ -128,16 +128,17 @@ EXTRA_JS_MODULES += [
     'BrowserErrorReporter.jsm',
     'BrowserUsageTelemetry.jsm',
     'BrowserWindowTracker.jsm',
     'ContentClick.jsm',
     'ContentCrashHandlers.jsm',
     'ContentMetaHandler.jsm',
     'ContentObservers.js',
     'ContentSearch.jsm',
+    'Discovery.jsm',
     'ExtensionsUI.jsm',
     'FaviconLoader.jsm',
     'Feeds.jsm',
     'FormValidationHandler.jsm',
     'HomePage.jsm',
     'LaterRun.jsm',
     'LiveBookmarkMigrator.jsm',
     'OpenInTabsUtils.jsm',
new file mode 100644
--- /dev/null
+++ b/browser/modules/test/unit/test_discovery.js
@@ -0,0 +1,101 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+/* globals ChromeUtils, Assert, add_task */
+"use strict";
+
+// ClientID fails without...
+do_get_profile();
+
+ChromeUtils.import("resource://gre/modules/Services.jsm");
+ChromeUtils.import("resource://testing-common/TestUtils.jsm");
+ChromeUtils.import("resource://gre/modules/ClientID.jsm");
+ChromeUtils.import("resource:///modules/Discovery.jsm");
+ChromeUtils.import("resource://gre/modules/ContextualIdentityService.jsm");
+
+const TAAR_COOKIE_NAME = "taarId";
+
+add_task(async function test_discovery() {
+  let uri = Services.io.newURI("https://example.com/foobar");
+
+  // Ensure the prefs we need
+  Services.prefs.setBoolPref("browser.discovery.enabled", true);
+  Services.prefs.setBoolPref("browser.discovery.containers.enabled", true);
+  Services.prefs.setBoolPref("datareporting.healthreport.uploadEnabled", true);
+  Services.prefs.setCharPref("browser.discovery.sites", uri.host);
+
+  registerCleanupFunction(() => {
+    Services.prefs.clearUserPref("browser.discovery.enabled");
+    Services.prefs.clearUserPref("browser.discovery.containers.enabled");
+    Services.prefs.clearUserPref("browser.discovery.sites");
+    Services.prefs.clearUserPref("datareporting.healthreport.uploadEnabled");
+  });
+
+  // This is normally initialized by telemetry, force id creation.  This results
+  // in Discovery setting the cookie.
+  await ClientID.getClientID();
+  await Discovery.update();
+
+  ok(Services.cookies.cookieExists(uri.host, "/", TAAR_COOKIE_NAME, {}), "cookie exists");
+  ok(!Services.cookies.cookieExists(uri.host, "/", TAAR_COOKIE_NAME, {privateBrowsingId: 1}), "no private cookie exists");
+  ContextualIdentityService.getPublicIdentities().forEach(identity => {
+    let {userContextId} = identity;
+    equal(Services.cookies.cookieExists(uri.host, "/", TAAR_COOKIE_NAME, {userContextId}), identity.public, "cookie exists");
+  });
+
+  // Test the addition of a new container.
+  let changed = TestUtils.topicObserved("cookie-changed", (subject, data) => {
+    let cookie = subject.QueryInterface(Ci.nsICookie2);
+    equal(cookie.name, TAAR_COOKIE_NAME, "taar cookie exists");
+    equal(cookie.host, uri.host, "cookie exists for host");
+    equal(cookie.originAttributes.userContextId, container.userContextId, "cookie userContextId is correct");
+    return true;
+  });
+  let container = ContextualIdentityService.create("New Container", "Icon", "Color");
+  await changed;
+
+  // Test disabling
+  Discovery.enabled = false;
+  // Wait for the update to remove the cookie.
+  await TestUtils.waitForCondition(() => {
+    return !Services.cookies.cookieExists(uri.host, "/", TAAR_COOKIE_NAME, {});
+  });
+
+  ContextualIdentityService.getPublicIdentities().forEach(identity => {
+    let {userContextId} = identity;
+    ok(!Services.cookies.cookieExists(uri.host, "/", TAAR_COOKIE_NAME, {userContextId}), "no cookie exists");
+  });
+
+  // turn off containers
+  Services.prefs.setBoolPref("browser.discovery.containers.enabled", false);
+
+  Discovery.enabled = true;
+  await TestUtils.waitForCondition(() => {
+    return Services.cookies.cookieExists(uri.host, "/", TAAR_COOKIE_NAME, {});
+  });
+  // make sure we did not set cookies on containers
+  ContextualIdentityService.getPublicIdentities().forEach(identity => {
+    let {userContextId} = identity;
+    ok(!Services.cookies.cookieExists(uri.host, "/", TAAR_COOKIE_NAME, {userContextId}), "no cookie exists");
+  });
+
+  // Make sure clientId changes update discovery
+  changed = TestUtils.topicObserved("cookie-changed", (subject, data) => {
+    if (data !== "added") {
+      return false;
+    }
+    let cookie = subject.QueryInterface(Ci.nsICookie2);
+    equal(cookie.name, TAAR_COOKIE_NAME, "taar cookie exists");
+    equal(cookie.host, uri.host, "cookie exists for host");
+    return true;
+  });
+  ClientID.resetClientID();
+  await changed;
+
+  // Make sure disabling telemetry disables discovery.
+  Services.prefs.setBoolPref("datareporting.healthreport.uploadEnabled", false);
+  await TestUtils.waitForCondition(() => {
+    return !Services.cookies.cookieExists(uri.host, "/", TAAR_COOKIE_NAME, {});
+  });
+});
--- a/browser/modules/test/unit/xpcshell.ini
+++ b/browser/modules/test/unit/xpcshell.ini
@@ -4,8 +4,9 @@ firefox-appdir = browser
 skip-if = toolkit == 'android'
 
 [test_E10SUtils_nested_URIs.js]
 [test_HomePage.js]
 [test_LiveBookmarkMigrator.js]
 [test_Sanitizer_interrupted.js]
 [test_SitePermissions.js]
 [test_LaterRun.js]
+[test_discovery.js]
--- a/toolkit/modules/ClientID.jsm
+++ b/toolkit/modules/ClientID.jsm
@@ -16,16 +16,20 @@ const LOGGER_PREFIX = "ClientID::";
 // Must match ID in TelemetryUtils
 const CANARY_CLIENT_ID = "c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0";
 
 ChromeUtils.defineModuleGetter(this, "CommonUtils",
                                "resource://services-common/utils.js");
 ChromeUtils.defineModuleGetter(this, "OS",
                                "resource://gre/modules/osfile.jsm");
 
+XPCOMUtils.defineLazyGetter(this, "CryptoHash", () => {
+  return Components.Constructor("@mozilla.org/security/hash;1", "nsICryptoHash", "initWithString");
+});
+
 XPCOMUtils.defineLazyGetter(this, "gDatareportingPath", () => {
   return OS.Path.join(OS.Constants.Path.profileDir, "datareporting");
 });
 
 XPCOMUtils.defineLazyGetter(this, "gStateFilePath", () => {
   return OS.Path.join(gDatareportingPath, "state.json");
 });
 
@@ -77,16 +81,20 @@ var ClientID = Object.freeze({
    *  - the current on-disk client id if it was already loaded
    *  - the client id that we cached into preferences (if any)
    *  - null otherwise
    */
   getCachedClientID() {
     return ClientIDImpl.getCachedClientID();
   },
 
+  async getClientIdHash() {
+    return ClientIDImpl.getClientIdHash();
+  },
+
   /**
    * Set a specific client id asynchronously, writing it to disk
    * and updating the cached version.
    *
    * Should only ever be used when a known client ID value should be set.
    * Use `resetClientID` to generate a new random one if required.
    *
    * @return {Promise<string>} The stable client ID.
@@ -113,16 +121,17 @@ var ClientID = Object.freeze({
    */
   _reset() {
     return ClientIDImpl._reset();
   },
 });
 
 var ClientIDImpl = {
   _clientID: null,
+  _clientIDHash: null,
   _loadClientIdTask: null,
   _saveClientIdTask: null,
   _removeClientIdTask: null,
   _logger: null,
   _wasCanary: null,
 
   _loadClientID() {
     if (this._loadClientIdTask) {
@@ -238,38 +247,50 @@ var ClientIDImpl = {
     if (!isValidClientID(id)) {
       this._log.error("getCachedClientID - invalid client id in preferences, resetting", id);
       Services.prefs.clearUserPref(PREF_CACHED_CLIENTID);
       return null;
     }
     return id;
   },
 
+  async getClientIdHash() {
+    if (!this._clientIDHash) {
+      let byteArr = new TextEncoder().encode(await this.getClientID());
+      let hash = new CryptoHash("sha256");
+      hash.update(byteArr, byteArr.length);
+      this._clientIDHash = CommonUtils.bytesAsHex(hash.finish(false));
+    }
+    return this._clientIDHash;
+  },
+
   /*
    * Resets the provider. This is for testing only.
    */
   async _reset() {
     await this._loadClientIdTask;
     await this._saveClientIdTask;
     this._clientID = null;
+    this._clientIDHash = null;
   },
 
   async setClientID(id) {
     if (!this.updateClientID(id)) {
       throw ("Invalid client ID: " + id);
     }
 
     this._saveClientIdTask = this._saveClientID();
     await this._saveClientIdTask;
     return this._clientID;
   },
 
   async _doRemoveClientID() {
     // Reset stored id.
     this._clientID = null;
+    this._clientIDHash = null;
 
     // Clear the client id from the preference cache.
     Services.prefs.clearUserPref(PREF_CACHED_CLIENTID);
 
     // Remove the client id from disk
     await OS.File.remove(gStateFilePath, {ignoreAbsent: true});
   },
 
@@ -303,16 +324,17 @@ var ClientIDImpl = {
    */
   updateClientID(id) {
     if (!isValidClientID(id)) {
       this._log.error("updateClientID - invalid client ID", id);
       return false;
     }
 
     this._clientID = id;
+    this._clientIDHash = null;
     Services.prefs.setStringPref(PREF_CACHED_CLIENTID, this._clientID);
     return true;
   },
 
   /**
    * A helper for getting access to telemetry logger.
    */
   get _log() {