Bug 1467705 - Refactor the histogram packing routines in TelemetryUtils. r=chutten
authorAlessio Placitelli <alessio.placitelli@gmail.com>
Fri, 08 Jun 2018 18:44:31 +0200
changeset 422610 fc7a9042ec08b85734714d4f7bffb4e34ca0dd85
parent 422609 3774e59affc5fed9d9bcff7ecce0f1c46df4c738
child 422611 bde911a9320ae5844f01f558ffc3670d5c0fe24d
push id34139
push useraciure@mozilla.com
push dateFri, 15 Jun 2018 09:48:05 +0000
treeherdermozilla-central@dc997a4e045e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschutten
bugs1467705
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1467705 - Refactor the histogram packing routines in TelemetryUtils. r=chutten Firefox Desktop is not sending raw histograms out, in pings. Instead, it processes the raw histograms and packs them in a sparser representation. GeckoView will need to do the same, so share the code rather than copy paste it. MozReview-Commit-ID: IW3tSHC12fc
toolkit/components/telemetry/TelemetrySession.jsm
toolkit/components/telemetry/TelemetryUtils.jsm
--- a/toolkit/components/telemetry/TelemetrySession.jsm
+++ b/toolkit/components/telemetry/TelemetrySession.jsm
@@ -807,113 +807,32 @@ var Impl = {
     ret.activeTicks = activeTicks;
 
     ret.pingsOverdue = TelemetrySend.overduePingsCount;
 
     return ret;
   },
 
   /**
-   * When reflecting a histogram into JS, Telemetry hands us an object
-   * with the following properties:
-   *
-   * - min, max, histogram_type, sum, sum_squares_{lo,hi}: simple integers;
-   * - counts: array of counts for histogram buckets;
-   * - ranges: array of calculated bucket sizes.
-   *
-   * This format is not straightforward to read and potentially bulky
-   * with lots of zeros in the counts array.  Packing histograms makes
-   * raw histograms easier to read and compresses the data a little bit.
-   *
-   * Returns an object:
-   * { range: [min, max], bucket_count: <number of buckets>,
-   *   histogram_type: <histogram_type>, sum: <sum>,
-   *   values: { bucket1: count1, bucket2: count2, ... } }
-   */
-  packHistogram: function packHistogram(hgram) {
-    let r = hgram.ranges;
-    let c = hgram.counts;
-    let retgram = {
-      range: [r[1], r[r.length - 1]],
-      bucket_count: r.length,
-      histogram_type: hgram.histogram_type,
-      values: {},
-      sum: hgram.sum
-    };
-
-    let first = true;
-    let last = 0;
-
-    for (let i = 0; i < c.length; i++) {
-      let value = c[i];
-      if (!value)
-        continue;
-
-      // add a lower bound
-      if (i && first) {
-        retgram.values[r[i - 1]] = 0;
-      }
-      first = false;
-      last = i + 1;
-      retgram.values[r[i]] = value;
-    }
-
-    // add an upper bound
-    if (last && last < c.length)
-      retgram.values[r[last]] = 0;
-    return retgram;
-  },
-
-  /**
    * Get the type of the dataset that needs to be collected, based on the preferences.
    * @return {Integer} A value from nsITelemetry.DATASET_*.
    */
   getDatasetType() {
     return Telemetry.canRecordExtended ? Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN
                                        : Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTOUT;
   },
 
   getHistograms: function getHistograms(clearSubsession) {
     let hls = Telemetry.snapshotHistograms(this.getDatasetType(), clearSubsession);
-    let ret = {};
-
-    for (let [process, histograms] of Object.entries(hls)) {
-      ret[process] = {};
-      for (let [name, value] of Object.entries(histograms)) {
-        if (this._testing || !name.startsWith("TELEMETRY_TEST_")) {
-          ret[process][name] = this.packHistogram(value);
-        }
-      }
-    }
-
-    return ret;
+    return TelemetryUtils.packHistograms(hls, this._testing);
   },
 
   getKeyedHistograms(clearSubsession) {
     let khs = Telemetry.snapshotKeyedHistograms(this.getDatasetType(), clearSubsession);
-    let ret = {};
-
-    for (let [process, histograms] of Object.entries(khs)) {
-      ret[process] = {};
-      for (let [name, value] of Object.entries(histograms)) {
-        if (this._testing || !name.startsWith("TELEMETRY_TEST_")) {
-          let keys = Object.keys(value);
-          if (keys.length == 0) {
-            // Skip empty keyed histogram
-            continue;
-          }
-          ret[process][name] = {};
-          for (let [key, hgram] of Object.entries(value)) {
-            ret[process][name][key] = this.packHistogram(hgram);
-          }
-        }
-      }
-    }
-
-    return ret;
+    return TelemetryUtils.packKeyedHistograms(khs, this._testing);
   },
 
   /**
    * Get a snapshot of the scalars and clear them.
    * @param {subsession} If true, then we collect the data for a subsession.
    * @param {clearSubsession} If true, we  need to clear the subsession.
    * @param {keyed} Take a snapshot of keyed or non keyed scalars.
    * @return {Object} The scalar data as a Javascript object, including the
--- a/toolkit/components/telemetry/TelemetryUtils.jsm
+++ b/toolkit/components/telemetry/TelemetryUtils.jsm
@@ -19,16 +19,67 @@ const PREF_TELEMETRY_ENABLED = "toolkit.
 const IS_CONTENT_PROCESS = (function() {
   // We cannot use Services.appinfo here because in telemetry xpcshell tests,
   // appinfo is initially unavailable, and becomes available only later on.
   // eslint-disable-next-line mozilla/use-services
   let runtime = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime);
   return runtime.processType == Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT;
 })();
 
+/**
+ * When reflecting a histogram into JS, Telemetry hands us an object
+ * with the following properties:
+ *
+ * - min, max, histogram_type, sum, sum_squares_{lo,hi}: simple integers;
+ * - counts: array of counts for histogram buckets;
+ * - ranges: array of calculated bucket sizes.
+ *
+ * This format is not straightforward to read and potentially bulky
+ * with lots of zeros in the counts array.  Packing histograms makes
+ * raw histograms easier to read and compresses the data a little bit.
+ *
+ * Returns an object:
+ * { range: [min, max], bucket_count: <number of buckets>,
+ *   histogram_type: <histogram_type>, sum: <sum>,
+ *   values: { bucket1: count1, bucket2: count2, ... } }
+ */
+function packHistogram(hgram) {
+  let r = hgram.ranges;
+  let c = hgram.counts;
+  let retgram = {
+    range: [r[1], r[r.length - 1]],
+    bucket_count: r.length,
+    histogram_type: hgram.histogram_type,
+    values: {},
+    sum: hgram.sum
+  };
+
+  let first = true;
+  let last = 0;
+
+  for (let i = 0; i < c.length; i++) {
+    let value = c[i];
+    if (!value)
+      continue;
+
+    // add a lower bound
+    if (i && first) {
+      retgram.values[r[i - 1]] = 0;
+    }
+    first = false;
+    last = i + 1;
+    retgram.values[r[i]] = value;
+  }
+
+  // add an upper bound
+  if (last && last < c.length)
+    retgram.values[r[last]] = 0;
+  return retgram;
+}
+
 var TelemetryUtils = {
   Preferences: Object.freeze({
     // General Preferences
     ArchiveEnabled: "toolkit.telemetry.archive.enabled",
     CachedClientId: "toolkit.telemetry.cachedClientID",
     FirstRun: "toolkit.telemetry.reportingpolicy.firstRun",
     FirstShutdownPingEnabled: "toolkit.telemetry.firstShutdownPing.enabled",
     HealthPingEnabled: "toolkit.telemetry.healthping.enabled",
@@ -221,9 +272,97 @@ var TelemetryUtils = {
     const isReleaseCandidateOnBeta =
       AppConstants.MOZ_UPDATE_CHANNEL === "release" &&
       Services.prefs.getCharPref("app.update.channel", null) === "beta";
     Services.telemetry.canRecordBase = true;
     Services.telemetry.canRecordExtended = isPrereleaseChannel ||
       isReleaseCandidateOnBeta ||
       Services.prefs.getBoolPref(this.Preferences.OverridePreRelease, false);
   },
+
+  /**
+   * Converts histograms from the raw to the packed format.
+   * This additionally filters TELEMETRY_TEST_ histograms.
+   *
+   * @param {Object} snapshot - The histogram snapshot.
+   * @param {Boolean} [testingMode=false] - Whether or not testing histograms
+   *        should be filtered.
+   * @returns {Object}
+   *
+   * {
+   *  "<process>": {
+   *    "<histogram>": {
+   *      range: [min, max],
+   *      bucket_count: <number of buckets>,
+   *      histogram_type: <histogram_type>,
+   *      sum: <sum>,
+   *      values: { bucket1: count1, bucket2: count2, ... }
+   *    },
+   *   ..
+   *   },
+   *  ..
+   * }
+   */
+  packHistograms(snapshot, testingMode = false) {
+    let ret = {};
+
+    for (let [process, histograms] of Object.entries(snapshot)) {
+      ret[process] = {};
+      for (let [name, value] of Object.entries(histograms)) {
+        if (testingMode || !name.startsWith("TELEMETRY_TEST_")) {
+          ret[process][name] = packHistogram(value);
+        }
+      }
+    }
+
+    return ret;
+  },
+
+  /**
+   * Converts keyed histograms from the raw to the packed format.
+   * This additionally filters TELEMETRY_TEST_ histograms and skips
+   * empty keyed histograms.
+   *
+   * @param {Object} snapshot - The keyed histogram snapshot.
+   * @param {Boolean} [testingMode=false] - Whether or not testing histograms should
+   *        be filtered.
+   * @returns {Object}
+   *
+   * {
+   *  "<process>": {
+   *    "<histogram>": {
+   *      "<key>": {
+   *        range: [min, max],
+   *        bucket_count: <number of buckets>,
+   *        histogram_type: <histogram_type>,
+   *        sum: <sum>,
+   *        values: { bucket1: count1, bucket2: count2, ... }
+   *      },
+   *      ..
+   *    },
+   *   ..
+   *   },
+   *  ..
+   * }
+   */
+  packKeyedHistograms(snapshot, testingMode = false) {
+    let ret = {};
+
+    for (let [process, histograms] of Object.entries(snapshot)) {
+      ret[process] = {};
+      for (let [name, value] of Object.entries(histograms)) {
+        if (testingMode || !name.startsWith("TELEMETRY_TEST_")) {
+          let keys = Object.keys(value);
+          if (keys.length == 0) {
+            // Skip empty keyed histogram
+            continue;
+          }
+          ret[process][name] = {};
+          for (let [key, hgram] of Object.entries(value)) {
+            ret[process][name][key] = packHistogram(hgram);
+          }
+        }
+      }
+    }
+
+    return ret;
+  },
 };