Bug 1120362 - Part 3 - Reset subsession histograms on telemetry payload collections. r=vladan
authorGeorg Fritzsche <georg.fritzsche@googlemail.com>
Wed, 25 Feb 2015 23:54:33 +0100
changeset 230880 357e9b9efd189a096a228f7907cee354d2dcf025
parent 230879 d48dc6e66dc46255e0c910ba5b6627eefa582459
child 230881 3ac1091f8f676f8ba75c5b761757f33c6c573b2b
push id28338
push usercbook@mozilla.com
push dateThu, 26 Feb 2015 11:01:28 +0000
treeherdermozilla-central@df3daecd381f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersvladan
bugs1120362
milestone39.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1120362 - Part 3 - Reset subsession histograms on telemetry payload collections. r=vladan
toolkit/components/telemetry/TelemetrySession.jsm
toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
--- a/toolkit/components/telemetry/TelemetrySession.jsm
+++ b/toolkit/components/telemetry/TelemetrySession.jsm
@@ -12,16 +12,17 @@ const Cu = Components.utils;
 
 Cu.import("resource://gre/modules/debug.js", this);
 Cu.import("resource://gre/modules/Log.jsm");
 Cu.import("resource://gre/modules/Services.jsm", this);
 Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
 Cu.import("resource://gre/modules/Promise.jsm", this);
 Cu.import("resource://gre/modules/DeferredTask.jsm", this);
 Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/Timer.jsm");
 
 const IS_CONTENT_PROCESS = (function() {
   // We cannot use Services.appinfo here because in telemetry xpcshell tests,
   // appinfo is initially unavailable, and becomes available only later on.
   let runtime = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime);
   return runtime.processType == Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT;
 })();
 
@@ -101,16 +102,33 @@ XPCOMUtils.defineLazyModuleGetter(this, 
 
 function generateUUID() {
   let str = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator).generateUUID().toString();
   // strip {}
   return str.substring(1, str.length - 1);
 }
 
 /**
+ * This is a policy object used to override behavior for testing.
+ */
+let Policy = {
+  now: () => new Date(),
+};
+
+/**
+ * Takes a date and returns it trunctated to a date with daily precision.
+ */
+function truncateToDays(date) {
+  return new Date(date.getFullYear(),
+                  date.getMonth(),
+                  date.getDate(),
+                  0, 0, 0, 0);
+}
+
+/**
  * Read current process I/O counters.
  */
 let processInfo = {
   _initialized: false,
   _IO_COUNTERS: null,
   _kernel32: null,
   _GetProcessIoCounters: null,
   _GetCurrentProcess: null,
@@ -164,20 +182,21 @@ this.TelemetrySession = Object.freeze({
    * Send a ping to a test server. Used only for testing.
    */
   testPing: function() {
     return Impl.testPing();
   },
   /**
    * Returns the current telemetry payload.
    * @param reason Optional, the reason to trigger the payload.
+   * @param clearSubsession Optional, whether to clear subsession specific data.
    * @returns Object
    */
-  getPayload: function(reason) {
-    return Impl.getPayload(reason);
+  getPayload: function(reason, clearSubsession = false) {
+    return Impl.getPayload(reason, clearSubsession);
   },
   /**
    * Save histograms to a file.
    * Used only for testing purposes.
    *
    * @param {nsIFile} aFile The file to load from.
    */
   testSaveHistograms: function(aFile) {
@@ -274,16 +293,18 @@ let Impl = {
   // The previous build ID, if this is the first run with a new build.
   // Undefined if this is not the first run, or the previous build ID is unknown.
   _previousBuildID: undefined,
   // Telemetry payloads sent by child processes.
   // Each element is in the format {source: <weak-ref>, payload: <object>},
   // where source is a weak reference to the child process,
   // and payload is the telemetry payload from that child process.
   _childTelemetry: [],
+  // Date of the last session split
+  _subsessionStartDate: null,
 
   /**
    * Gets a series of simple measurements (counters). At the moment, this
    * only returns startup data from nsIAppStartup.getStartupInfo().
    *
    * @return simple measurements as a dictionary.
    */
   getSimpleMeasurements: function getSimpleMeasurements(forSavedSession) {
@@ -445,22 +466,22 @@ let Impl = {
     }
 
     // add an upper bound
     if (last && last < c.length)
       retgram.values[r[last]] = 0;
     return retgram;
   },
 
-  getHistograms: function getHistograms(subsession) {
-    this._log.trace("getHistograms");
+  getHistograms: function getHistograms(subsession, clearSubsession) {
+    this._log.trace("getHistograms - subsession: " + subsession + ", clearSubsession: " + clearSubsession);
 
     let registered =
       Telemetry.registeredHistograms(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, []);
-    let hls = subsession ? Telemetry.snapshotSubsessionHistograms(false)
+    let hls = subsession ? Telemetry.snapshotSubsessionHistograms(clearSubsession)
                          : Telemetry.histogramSnapshots;
     let ret = {};
 
     for (let name of registered) {
       for (let n of [name, "STARTUP_" + name]) {
         if (n in hls) {
           ret[n] = this.packHistogram(hls[n]);
         }
@@ -484,27 +505,33 @@ let Impl = {
       }
       if (Object.keys(packedHistograms).length != 0)
         ret[addonName] = packedHistograms;
     }
 
     return ret;
   },
 
-  getKeyedHistograms: function(subsession) {
-    this._log.trace("getKeyedHistograms");
+  getKeyedHistograms: function(subsession, clearSubsession) {
+    this._log.trace("getKeyedHistograms - subsession: " + subsession + ", clearSubsession: " + clearSubsession);
 
     let registered =
       Telemetry.registeredKeyedHistograms(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, []);
     let ret = {};
 
     for (let id of registered) {
       ret[id] = {};
       let keyed = Telemetry.getKeyedHistogramById(id);
-      let snapshot = subsession ? keyed.subsessionSnapshot() : keyed.snapshot();
+      let snapshot = null;
+      if (subsession) {
+        snapshot = clearSubsession ? keyed.snapshotSubsessionAndClear()
+                                   : keyed.subsessionSnapshot();
+      } else {
+        snapshot = keyed.snapshot();
+      }
       for (let key of Object.keys(snapshot)) {
         ret[id][key] = this.packHistogram(snapshot[key]);
       }
     }
 
     return ret;
   },
 
@@ -537,16 +564,18 @@ let Impl = {
       OS: ai.OS,
       appVersion: ai.version, // TODO: In Environment, but needed for |submissionPath|
       appName: ai.name, // TODO: In Environment, but needed for |submissionPath|
       appBuildID: ai.appBuildID, // TODO: In Environment, but needed for |submissionPath|
       appUpdateChannel: UpdateChannel.get(), // TODO: In Environment, but needed for |submissionPath|
       platformBuildID: ai.platformBuildID,
       revision: HISTOGRAMS_FILE_VERSION,
       asyncPluginInit: Preferences.get(PREF_ASYNC_PLUGIN_INIT, false)
+
+      subsessionStartDate: truncateToDays(this._subsessionStartDate).toISOString(),
     };
 
     // In order to share profile data, the appName used for Metro Firefox is "Firefox",
     // (the same as desktop Firefox). We set it to "MetroFirefox" here in order to
     // differentiate telemetry pings sent by desktop vs. metro Firefox.
     if(Services.metro && Services.metro.immersive) {
       ret.appName = "MetroFirefox";
     }
@@ -707,33 +736,27 @@ let Impl = {
   },
 
   /**
    * Get the current session's payload using the provided
    * simpleMeasurements and info, which are typically obtained by a call
    * to |this.getSimpleMeasurements| and |this.getMetadata|,
    * respectively.
    */
-  assemblePayloadWithMeasurements: function(simpleMeasurements, info, reason) {
-    const classicReasons = [
-      "saved-session",
-      "idle-daily",
-      "gather-payload",
-      "test-ping",
-    ];
-    const isSubsession = classicReasons.indexOf(reason) == -1;
+  assemblePayloadWithMeasurements: function(simpleMeasurements, info, reason, clearSubsession) {
+    const isSubsession = !this._isClassicReason(reason);
     this._log.trace("assemblePayloadWithMeasurements - reason: " + reason +
                     ", submitting subsession data: " + isSubsession);
 
     // Payload common to chrome and content processes.
     let payloadObj = {
       ver: PAYLOAD_VERSION,
       simpleMeasurements: simpleMeasurements,
-      histograms: this.getHistograms(isSubsession),
-      keyedHistograms: this.getKeyedHistograms(isSubsession),
+      histograms: this.getHistograms(isSubsession, clearSubsession),
+      keyedHistograms: this.getKeyedHistograms(isSubsession, clearSubsession),
       chromeHangs: Telemetry.chromeHangs,
       threadHangStats: this.getThreadHangStats(Telemetry.threadHangStats),
       log: TelemetryLog.entries(),
     };
 
     if (IS_CONTENT_PROCESS) {
       return payloadObj;
     }
@@ -756,35 +779,42 @@ let Impl = {
     let clientID = TelemetryPing.clientID;
     if (clientID && Preferences.get(PREF_FHR_UPLOAD_ENABLED, false)) {
       payloadObj.clientID = clientID;
     }
 
     if (this._childTelemetry.length) {
       payloadObj.childPayloads = this.getChildPayloads();
     }
+
     return payloadObj;
   },
 
-  getSessionPayload: function getSessionPayload(reason) {
-    this._log.trace("getSessionPayload - Reason " + reason);
+  getSessionPayload: function getSessionPayload(reason, clearSubsession) {
+    this._log.trace("getSessionPayload - reason: " + reason + ", clearSubsession: " + clearSubsession);
+
     let measurements = this.getSimpleMeasurements(reason == "saved-session");
     let info = !IS_CONTENT_PROCESS ? this.getMetadata(reason) : null;
-    return this.assemblePayloadWithMeasurements(measurements, info, reason);
+    let payload = this.assemblePayloadWithMeasurements(measurements, info, reason, clearSubsession);
+
+    this._subsessionStartDate = Policy.now();
+
+    return payload;
   },
 
   /**
    * Send data to the server. Record success/send-time in histograms
    */
   send: function send(reason) {
     this._log.trace("send - Reason " + reason);
     // populate histograms one last time
     this.gatherMemory();
 
-    let payload = this.getSessionPayload(reason);
+    const isSubsession = !this._isClassicReason(reason);
+    let payload = this.getSessionPayload(reason, isSubsession);
     let options = {
       retentionDays: RETENTION_DAYS,
       addClientId: true,
       addEnvironment: true,
     };
     return TelemetryPing.send(PING_TYPE, payload, options);
   },
 
@@ -840,16 +870,19 @@ let Impl = {
    */
   setupChromeProcess: function setupChromeProcess(testing) {
     if (testing && !this._log) {
       this._log = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);
     }
 
     this._log.trace("setupChromeProcess");
 
+    this._sessionStartDate = Policy.now();
+    this._subsessionStartDate = this._sessionStartDate;
+
     // Initialize some probes that are kept in their own modules
     this._thirdPartyCookies = new ThirdPartyCookieProbe();
     this._thirdPartyCookies.init();
 
     // Record old value and update build ID preference if this is the first
     // run with a new build ID.
     let previousBuildID = Preferences.get(PREF_PREVIOUS_BUILDID, undefined);
     let thisBuildID = Services.appinfo.appBuildID;
@@ -962,34 +995,35 @@ let Impl = {
     }
     default:
       throw new Error("Telemetry.receiveMessage: bad message name");
     }
   },
 
   sendContentProcessPing: function sendContentProcessPing(reason) {
     this._log.trace("sendContentProcessPing - Reason " + reason);
-    let payload = this.getSessionPayload(reason);
+    const isSubsession = !this._isClassicReason(reason);
+    let payload = this.getSessionPayload(reason, isSubsession);
     cpmm.sendAsyncMessage(MESSAGE_TELEMETRY_PAYLOAD, payload);
   },
 
   savePendingPings: function savePendingPings() {
     this._log.trace("savePendingPings");
-    let payload = this.getSessionPayload("saved-session");
+    let payload = this.getSessionPayload("saved-session", false);
     let options = {
       retentionDays: RETENTION_DAYS,
       addClientId: true,
       addEnvironment: true,
     };
     return TelemetryPing.savePendingPings(PING_TYPE, payload, options);
   },
 
   testSaveHistograms: function testSaveHistograms(file) {
     this._log.trace("testSaveHistograms - Path: " + file.path);
-    let payload = this.getSessionPayload("saved-session");
+    let payload = this.getSessionPayload("saved-session", false);
     let options = {
       retentionDays: RETENTION_DAYS,
       addClientId: true,
       addEnvironment: true,
       overwrite: true,
       filePath: file.path,
     };
     return TelemetryPing.testSavePingToFile(PING_TYPE, payload, options);
@@ -1008,27 +1042,27 @@ let Impl = {
       Services.obs.removeObserver(this, "xul-window-visible");
       this._hasXulWindowVisibleObserver = false;
     }
 #ifdef MOZ_WIDGET_ANDROID
     Services.obs.removeObserver(this, "application-background", false);
 #endif
   },
 
-  getPayload: function getPayload(reason) {
-    this._log.trace("getPayload");
+  getPayload: function getPayload(reason, clearSubsession) {
+    this._log.trace("getPayload - clearSubsession: " + clearSubsession);
     reason = reason || "gather-payload";
     // This function returns the current Telemetry payload to the caller.
     // We only gather startup info once.
     if (Object.keys(this._slowSQLStartup).length == 0) {
       this.gatherStartupHistograms();
       this._slowSQLStartup = Telemetry.slowSQL;
     }
     this.gatherMemory();
-    return this.getSessionPayload(reason);
+    return this.getSessionPayload(reason, clearSubsession);
   },
 
   gatherStartup: function gatherStartup() {
     this._log.trace("gatherStartup");
     let counters = processInfo.getCounters();
     if (counters) {
       [this._startupIO.startupSessionRestoreReadBytes,
         this._startupIO.startupSessionRestoreWriteBytes] = counters;
@@ -1137,17 +1171,17 @@ let Impl = {
     // 3) We submit the data, and then come back into the foreground. Same as case (2).
     // 4) We do not submit the data, but come back into the foreground. In this case
     //    we have the option of either deleting the file that we saved (since we will either
     //    send the live data while in the foreground, or create the file again on the next
     //    backgrounding), or not (in which case we will delete it on submit, or overwrite
     //    it on the next backgrounding). Not deleting it is faster, so that's what we do.
     case "application-background":
       if (Telemetry.canSend) {
-        let payload = this.getSessionPayload("saved-session");
+        let payload = this.getSessionPayload("saved-session", false);
         let options = {
           retentionDays: RETENTION_DAYS,
           addClientId: true,
           addEnvironment: true,
           overwrite: true,
         };
         TelemetryPing.savePing(PING_TYPE, payload, options);
       }
@@ -1163,9 +1197,19 @@ let Impl = {
    */
   shutdown: function(testing = false) {
     this.uninstall();
     if (Telemetry.canSend || testing) {
       return this.savePendingPings();
     }
     return Promise.resolve();
   },
+
+  _isClassicReason: function(reason) {
+    const classicReasons = [
+      "saved-session",
+      "idle-daily",
+      "gather-payload",
+      "test-ping",
+    ];
+    return classicReasons.indexOf(reason) != -1;
+  },
 };
--- a/toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
@@ -91,16 +91,21 @@ function wrapWithExceptionHandler(f) {
       dump("Caught exception: " + ex.message + "\n");
       dump(ex.stack);
       do_test_finished();
     }
   }
   return wrapper;
 }
 
+function fakeNow(date) {
+  let session = Cu.import("resource://gre/modules/TelemetrySession.jsm");
+  session.Policy.now = () => date;
+}
+
 function registerPingHandler(handler) {
   gHttpServer.registerPrefixHandler("/submit/telemetry/",
 				   wrapWithExceptionHandler(handler));
 }
 
 function setupTestData() {
   Telemetry.newHistogram(IGNORE_HISTOGRAM, "never", Telemetry.HISTOGRAM_BOOLEAN);
   Telemetry.histogramFrom(IGNORE_CLONED_HISTOGRAM, IGNORE_HISTOGRAM_TO_CLONE);
@@ -533,16 +538,20 @@ add_task(function* test_saveLoadPing() {
     checkPayload(ping2.payload, "saved-session", 1);
   } else {
     checkPayload(ping1.payload, "saved-session", 1);
     checkPayload(ping2.payload, "test-ping", 1);
   }
 });
 
 add_task(function* test_checkSubsession() {
+  let now = new Date(2020, 1, 1, 12, 0, 0);
+  let expectedDate = new Date(2020, 1, 1, 0, 0, 0);
+  fakeNow(now);
+
   const COUNT_ID = "TELEMETRY_TEST_COUNT";
   const KEYED_ID = "TELEMETRY_TEST_KEYED_COUNT";
   const count = Telemetry.getHistogramById(COUNT_ID);
   const keyed = Telemetry.getKeyedHistogramById(KEYED_ID);
   const registeredIds =
     new Set(Telemetry.registeredHistograms(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, []));
 
   const stableHistograms = new Set([
@@ -663,18 +672,27 @@ add_task(function* test_checkSubsession(
   Assert.equal(classic.histograms[COUNT_ID].sum, 1);
   Assert.equal(classic.keyedHistograms[KEYED_ID]["a"].sum, 1);
   Assert.equal(classic.keyedHistograms[KEYED_ID]["b"].sum, 1);
 
   checkHistograms(classic.histograms, subsession.histograms);
   checkKeyedHistograms(classic.keyedHistograms, subsession.keyedHistograms);
 
   // We should be able to reset only the subsession histograms.
-  count.clear(true);
-  keyed.clear(true);
+  // First check that "snapshot and clear" still returns the old state...
+  classic = TelemetrySession.getPayload();
+  subsession = TelemetrySession.getPayload("environment-change", true);
+
+  Assert.equal(classic.info.subsessionStartDate, expectedDate.toISOString());
+  Assert.equal(subsession.info.subsessionStartDate, expectedDate.toISOString());
+  checkHistograms(classic.histograms, subsession.histograms);
+  checkKeyedHistograms(classic.keyedHistograms, subsession.keyedHistograms);
+
+  // ... then check that the next snapshot shows the subsession
+  // histograms got reset.
   classic = TelemetrySession.getPayload();
   subsession = TelemetrySession.getPayload("environment-change");
 
   Assert.ok(COUNT_ID in classic.histograms);
   Assert.ok(COUNT_ID in subsession.histograms);
   Assert.equal(classic.histograms[COUNT_ID].sum, 1);
   Assert.equal(subsession.histograms[COUNT_ID].sum, 0);