Bug 1191912 - Part 2 - Enable opt-out Telemetry for a 5% sample of release users. r=rvitillo,a=ritu
authorGeorg Fritzsche <georg.fritzsche@googlemail.com>
Tue, 11 Aug 2015 12:17:30 +0200
changeset 288730 86a5a7e54c8f7fddbf3b0f00de37ba94d43f2e99
parent 288729 463fe048a77808672970ccbde58faa7670afab30
child 288731 ffddd1f9cf27c44a2749629d052a1cf9da06242c
push id5067
push userraliiev@mozilla.com
push dateMon, 21 Sep 2015 14:04:52 +0000
treeherdermozilla-beta@14221ffe5b2f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrvitillo, ritu
bugs1191912
milestone42.0a2
Bug 1191912 - Part 2 - Enable opt-out Telemetry for a 5% sample of release users. r=rvitillo,a=ritu
toolkit/components/telemetry/TelemetryController.jsm
toolkit/components/telemetry/TelemetryEnvironment.jsm
toolkit/components/telemetry/docs/environment.rst
toolkit/components/telemetry/tests/unit/head.js
toolkit/components/telemetry/tests/unit/test_TelemetryController.js
toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
--- a/toolkit/components/telemetry/TelemetryController.jsm
+++ b/toolkit/components/telemetry/TelemetryController.jsm
@@ -36,16 +36,17 @@ const PREF_ENABLED = PREF_BRANCH + "enab
 const PREF_LOG_LEVEL = PREF_BRANCH_LOG + "level";
 const PREF_LOG_DUMP = PREF_BRANCH_LOG + "dump";
 const PREF_CACHED_CLIENTID = PREF_BRANCH + "cachedClientID";
 const PREF_FHR_ENABLED = "datareporting.healthreport.service.enabled";
 const PREF_FHR_UPLOAD_ENABLED = "datareporting.healthreport.uploadEnabled";
 const PREF_SESSIONS_BRANCH = "datareporting.sessions.";
 const PREF_UNIFIED = PREF_BRANCH + "unified";
 const PREF_UNIFIED_OPTIN = PREF_BRANCH + "unifiedIsOptIn";
+const PREF_OPTOUT_SAMPLE = PREF_BRANCH + "optoutSample";
 
 // Whether the FHR/Telemetry unification features are enabled.
 // Changing this pref requires a restart.
 const IS_UNIFIED_TELEMETRY = Preferences.get(PREF_UNIFIED, false);
 // This preference allows to leave unified Telemetry behavior on only for people that
 // opted into Telemetry. Changing this pref requires a restart.
 const IS_UNIFIED_OPTIN = Preferences.get(PREF_UNIFIED_OPTIN, false);
 
@@ -85,16 +86,39 @@ XPCOMUtils.defineLazyModuleGetter(this, 
                                   "resource://gre/modules/TelemetryArchive.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "TelemetrySession",
                                   "resource://gre/modules/TelemetrySession.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "TelemetrySend",
                                   "resource://gre/modules/TelemetrySend.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "TelemetryReportingPolicy",
                                   "resource://gre/modules/TelemetryReportingPolicy.jsm");
 
+XPCOMUtils.defineLazyGetter(this, "gCrcTable", function() {
+  let c;
+  let table = [];
+  for (let n = 0; n < 256; n++) {
+      c = n;
+      for (let k =0; k < 8; k++) {
+          c = ((c&1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+      }
+      table[n] = c;
+  }
+  return table;
+});
+
+function crc32(str) {
+    let crc = 0 ^ (-1);
+
+    for (let i = 0; i < str.length; i++ ) {
+        crc = (crc >>> 8) ^ gCrcTable[(crc ^ str.charCodeAt(i)) & 0xFF];
+    }
+
+    return (crc ^ (-1)) >>> 0;
+}
+
 /**
  * Setup Telemetry logging. This function also gets called when loggin related
  * preferences change.
  */
 let gLogger = null;
 let gLogAppenderDump = null;
 function configureLogging() {
   if (!gLogger) {
@@ -124,16 +148,18 @@ function configureLogging() {
 }
 
 /**
  * This is a policy object used to override behavior for testing.
  */
 let Policy = {
   now: () => new Date(),
   generatePingId: () => Utils.generateUUID(),
+  getCachedClientID: () => ClientID.getCachedClientID(),
+  isUnifiedOptin: () => IS_UNIFIED_OPTIN,
 }
 
 this.EXPORTED_SYMBOLS = ["TelemetryController"];
 
 this.TelemetryController = Object.freeze({
   Constants: Object.freeze({
     PREF_ENABLED: PREF_ENABLED,
     PREF_LOG_LEVEL: PREF_LOG_LEVEL,
@@ -300,16 +326,25 @@ this.TelemetryController = Object.freeze
    *
    * @return The client id as string, or null.
    */
   get clientID() {
     return Impl.clientID;
   },
 
   /**
+   * Whether this client is part of a sample that gets opt-out Telemetry.
+   *
+   * @return {Boolean} Whether the client is part of the opt-out sample.
+   */
+  get isInOptoutSample() {
+    return Impl.isInOptoutSample;
+  },
+
+  /**
    * The AsyncShutdown.Barrier to synchronize with TelemetryController shutdown.
    */
   get shutdown() {
     return Impl._shutdownBarrier.client;
   },
 
   /**
    * The session recorder instance managed by Telemetry.
@@ -583,16 +618,44 @@ let Impl = {
     return TelemetryStorage.saveAbortedSessionPing(pingData);
   },
 
   removeAbortedSessionPing: function() {
     return TelemetryStorage.removeAbortedSessionPing();
   },
 
   /**
+   *
+   */
+  _isInOptoutSample: function() {
+    if (!Preferences.get(PREF_OPTOUT_SAMPLE, false)) {
+      this._log.config("_sampleForOptoutTelemetry - optout sampling is disabled");
+      return false;
+    }
+
+    const clientId = Policy.getCachedClientID();
+    if (!clientId) {
+      this._log.config("_sampleForOptoutTelemetry - no cached client id available")
+      return false;
+    }
+
+    // This mimics the server-side 1% sampling, so that we can get matching populations.
+    // The server samples on ((crc32(clientId) % 100) == 42), we match 42+X here to get
+    // a bigger sample.
+    const sample = crc32(clientId) % 100;
+    const offset = 42;
+    const range = 5; // sampling from 5%
+
+    const optout = (sample >= offset && sample < (offset + range));
+    this._log.config("_sampleForOptoutTelemetry - sampling for optout Telemetry - " +
+                     "offset: " + offset + ", range: " + range + ", sample: " + sample);
+    return optout;
+  },
+
+  /**
    * Perform telemetry initialization for either chrome or content process.
    * @return {Boolean} True if Telemetry is allowed to record at least base (FHR) data,
    *                   false otherwise.
    */
   enableTelemetryRecording: function enableTelemetryRecording() {
     // The thumbnail service also runs in a content process, even with e10s off.
     // We need to check if e10s is on so we don't submit child payloads for it.
     // We still need xpcshell child tests to work, so we skip this if test mode is enabled.
@@ -600,19 +663,21 @@ let Impl = {
       this._log.config("enableTelemetryRecording - not enabling Telemetry for non-e10s child process");
       Telemetry.canRecordBase = false;
       Telemetry.canRecordExtended = false;
       return false;
     }
 
     // Configure base Telemetry recording.
     // Unified Telemetry makes it opt-out unless the unifedOptin pref is set.
+    // Additionally, we make Telemetry opt-out for a 5% sample.
     // If extended Telemetry is enabled, base recording is always on as well.
     const enabled = Preferences.get(PREF_ENABLED, false);
-    Telemetry.canRecordBase = enabled || (IS_UNIFIED_TELEMETRY && !IS_UNIFIED_OPTIN);
+    const isOptout = IS_UNIFIED_TELEMETRY && (!Policy.isUnifiedOptin() || this._isInOptoutSample());
+    Telemetry.canRecordBase = enabled || isOptout;
 
 #ifdef MOZILLA_OFFICIAL
     // Enable extended telemetry if:
     //  * the telemetry preference is set and
     //  * this is an official build or we are in test-mode
     // We only do the latter check for official builds so that e.g. developer builds
     // still enable Telemetry based on prefs.
     Telemetry.canRecordExtended = enabled && (Telemetry.isOfficialTelemetry || this._testMode);
@@ -810,16 +875,20 @@ let Impl = {
       break;
     }
   },
 
   get clientID() {
     return this._clientID;
   },
 
+  get isInOptoutSample() {
+    return this._isInOptoutSample();
+  },
+
   /**
    * Get an object describing the current state of this module for AsyncShutdown diagnostics.
    */
   _getState: function() {
     return {
       initialized: this._initialized,
       initStarted: this._initStarted,
       haveDelayedInitTask: !!this._delayedInitTask,
--- a/toolkit/components/telemetry/TelemetryEnvironment.jsm
+++ b/toolkit/components/telemetry/TelemetryEnvironment.jsm
@@ -14,16 +14,17 @@ const myScope = this;
 Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
 Cu.import("resource://gre/modules/Task.jsm");
 Cu.import("resource://gre/modules/Log.jsm");
 Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://gre/modules/PromiseUtils.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/TelemetryUtils.jsm", this);
 Cu.import("resource://gre/modules/ObjectUtils.jsm");
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
 
 const Utils = TelemetryUtils;
 
 XPCOMUtils.defineLazyModuleGetter(this, "ctypes",
                                   "resource://gre/modules/ctypes.jsm");
 #ifndef MOZ_WIDGET_GONK
 Cu.import("resource://gre/modules/AddonManager.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "LightweightThemeManager",
@@ -996,16 +997,17 @@ EnvironmentCache.prototype = {
 
     this._currentEnvironment.settings = {
       blocklistEnabled: Preferences.get(PREF_BLOCKLIST_ENABLED, true),
 #ifndef MOZ_WIDGET_ANDROID
       isDefaultBrowser: this._isDefaultBrowser(),
 #endif
       e10sEnabled: Services.appinfo.browserTabsRemoteAutostart,
       telemetryEnabled: Preferences.get(PREF_TELEMETRY_ENABLED, false),
+      isInOptoutSample: TelemetryController.isInOptoutSample,
       locale: getBrowserLocale(),
       update: {
         channel: updateChannel,
         enabled: Preferences.get(PREF_UPDATE_ENABLED, true),
         autoDownload: Preferences.get(PREF_UPDATE_AUTODOWNLOAD, true),
       },
       userPrefs: this._getPrefData(),
     };
--- a/toolkit/components/telemetry/docs/environment.rst
+++ b/toolkit/components/telemetry/docs/environment.rst
@@ -37,16 +37,17 @@ Structure::
         defaultSearchEngine: <string>, // e.g. "yahoo"
         defaultSearchEngineData: {, // data about the current default engine
           name: <string>, // engine name, e.g. "Yahoo"; or "NONE" if no default
           loadPath: <string>, // where the engine line is located; missing if no default
           submissionURL: <string> // missing if no default or for user-installed engines
         },
         e10sEnabled: <bool>, // whether e10s is on, i.e. browser tabs open by default in a different process
         telemetryEnabled: <bool>, // false on failure
+        isInOptoutSample: <bool>, // whether this client is part of the opt-out sample
         locale: <string>, // e.g. "it", null on failure
         update: {
           channel: <string>, // e.g. "release", null on failure
           enabled: <bool>, // true on failure
           autoDownload: <bool>, // true on failure
         },
         userPrefs: {
           // Only prefs which are changed from the default value are listed
--- a/toolkit/components/telemetry/tests/unit/head.js
+++ b/toolkit/components/telemetry/tests/unit/head.js
@@ -269,16 +269,26 @@ function fakeMidnightPingFuzzingDelay(de
   module.Policy.midnightPingFuzzingDelay = () => delayMs;
 }
 
 function fakeGeneratePingId(func) {
   let module = Cu.import("resource://gre/modules/TelemetryController.jsm");
   module.Policy.generatePingId = func;
 }
 
+function fakeCachedClientId(uuid) {
+  let module = Cu.import("resource://gre/modules/TelemetryController.jsm");
+  module.Policy.getCachedClientID = () => uuid;
+}
+
+function fakeIsUnifiedOptin(isOptin) {
+  let module = Cu.import("resource://gre/modules/TelemetryController.jsm");
+  module.Policy.isUnifiedOptin = () => isOptin;
+}
+
 // Return a date that is |offset| ms in the future from |date|.
 function futureDate(date, offset) {
   return new Date(date.getTime() + offset);
 }
 
 function truncateToDays(aMsec) {
   return Math.floor(aMsec / MILLISECONDS_PER_DAY);
 }
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryController.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryController.js
@@ -28,16 +28,17 @@ const APP_VERSION = "1";
 const APP_NAME = "XPCShell";
 
 const PREF_BRANCH = "toolkit.telemetry.";
 const PREF_ENABLED = PREF_BRANCH + "enabled";
 const PREF_ARCHIVE_ENABLED = PREF_BRANCH + "archive.enabled";
 const PREF_FHR_UPLOAD_ENABLED = "datareporting.healthreport.uploadEnabled";
 const PREF_FHR_SERVICE_ENABLED = "datareporting.healthreport.service.enabled";
 const PREF_UNIFIED = PREF_BRANCH + "unified";
+const PREF_OPTOUT_SAMPLE = PREF_BRANCH + "optoutSample";
 
 let gClientID = null;
 
 function sendPing(aSendClientId, aSendEnvironment) {
   if (PingServer.started) {
     TelemetrySend.setServer("http://localhost:" + PingServer.port);
   } else {
     TelemetrySend.setServer("http://doesnotexist");
@@ -363,12 +364,70 @@ add_task(function* test_changePingAfterS
   const pingId = yield pingPromise;
 
   // Make sure our changes didn't affect the submitted payload.
   let archivedCopy = yield TelemetryArchive.promiseArchivedPingById(pingId);
   Assert.equal(archivedCopy.payload.canary, "test",
                "The payload must not be changed after being submitted.");
 });
 
+add_task(function* test_optoutSampling() {
+  if (!Preferences.get(PREF_UNIFIED, false)) {
+    dump("Unified Telemetry is disabled, skipping.\n");
+    return;
+  }
+
+  const DATA = [
+    {uuid: null,                                   sampled: false}, // not to be sampled
+    {uuid: "3d38d821-14a4-3d45-ab0b-02a9fb5a7505", sampled: false}, // samples to 0
+    {uuid: "1331255e-7eb5-aa4f-b04e-494a0c6da282", sampled: false}, // samples to 41
+    {uuid: "35393e78-a363-ea4e-9fc9-9f9abbee2077", sampled: true }, // samples to 42
+    {uuid: "4dc81df6-db03-a34e-ba79-3e877afd22c4", sampled: true }, // samples to 43
+    {uuid: "79e15be6-4884-8d4f-98e5-f94790251e5f", sampled: true }, // samples to 44
+    {uuid: "c3841566-e39e-384d-826f-508ab6387b21", sampled: true }, // samples to 45
+    {uuid: "cc7498a4-2cde-da47-89b3-f3ce5dd7c6fc", sampled: true }, // samples to 46
+    {uuid: "0750d8ed-5969-3a4f-90ba-2e85f9074309", sampled: false}, // samples to 47
+    {uuid: "0dfcbce7-d82b-b144-8d77-eb15935c9a8e", sampled: false}, // samples to 99
+  ];
+
+  // Test that the opt-out pref enables us sampling on 5% of release.
+  Preferences.set(PREF_ENABLED, false);
+  Preferences.set(PREF_OPTOUT_SAMPLE, true);
+  fakeIsUnifiedOptin(true);
+
+  for (let d of DATA) {
+    dump("Testing sampling for uuid: " + d.uuid + "\n");
+    fakeCachedClientId(d.uuid);
+    yield TelemetryController.reset();
+    Assert.equal(TelemetryController.isInOptoutSample, d.sampled,
+                 "Opt-out sampling should behave as expected");
+    Assert.equal(Telemetry.canRecordBase, d.sampled,
+                 "Base recording setting should be correct");
+  }
+
+  // If we disable opt-out sampling Telemetry, have the opt-in setting on and extended Telemetry off,
+  // we should not enable anything.
+  Preferences.set(PREF_OPTOUT_SAMPLE, false);
+  fakeIsUnifiedOptin(true);
+  for (let d of DATA) {
+    dump("Testing sampling for uuid: " + d.uuid + "\n");
+    fakeCachedClientId(d.uuid);
+    yield TelemetryController.reset();
+    Assert.equal(Telemetry.canRecordBase, false,
+                 "Sampling should not override the default opt-out behavior");
+  }
+
+  // If we fully enable opt-out Telemetry on release, the sampling should not override that.
+  Preferences.set(PREF_OPTOUT_SAMPLE, true);
+  fakeIsUnifiedOptin(false);
+  for (let d of DATA) {
+    dump("Testing sampling for uuid: " + d.uuid + "\n");
+    fakeCachedClientId(d.uuid);
+    yield TelemetryController.reset();
+    Assert.equal(Telemetry.canRecordBase, true,
+                 "Sampling should not override the default opt-out behavior");
+  }
+});
+
 add_task(function* stopServer(){
   yield PingServer.stop();
   do_test_finished();
 });
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
@@ -250,16 +250,17 @@ function checkBuildSection(data) {
   }
 }
 
 function checkSettingsSection(data) {
   const EXPECTED_FIELDS_TYPES = {
     blocklistEnabled: "boolean",
     e10sEnabled: "boolean",
     telemetryEnabled: "boolean",
+    isInOptoutSample: "boolean",
     locale: "string",
     update: "object",
     userPrefs: "object",
   };
 
   Assert.ok("settings" in data, "There must be a settings section in Environment.");
 
   for (let f in EXPECTED_FIELDS_TYPES) {