Bug 1465251 - use PrioEncoder to encode Telemetry values for pilot project r=kmag
authorRobert Helmer <rhelmer@mozilla.com>
Sun, 09 Sep 2018 00:23:52 +0000
changeset 435538 8c55a3d07f0cf613fac4b45e51487fc8f6904f37
parent 435537 f4f0af49af15aea7484896dd856b2bceefb080c0
child 435539 4358bc28cad00015ebded7618e8d930ddedc19d1
push id34614
push userapavel@mozilla.com
push dateMon, 10 Sep 2018 21:59:18 +0000
treeherdermozilla-central@fea371cafd2c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerskmag
bugs1465251
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1465251 - use PrioEncoder to encode Telemetry values for pilot project r=kmag Use PrioEncoder to encode a few already-included histograms, so we can compare results on the Telemetry server side. Differential Revision: https://phabricator.services.mozilla.com/D5088
browser/app/profile/firefox.js
browser/moz.build
dom/chrome-webidl/PrioEncoder.webidl
dom/prio/PrioEncoder.cpp
dom/prio/PrioEncoder.h
modules/libpref/init/all.js
toolkit/components/telemetry/docs/data/main-ping.rst
toolkit/components/telemetry/pings/TelemetrySession.jsm
toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
toolkit/moz.configure
tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -1754,16 +1754,26 @@ pref("app.shield.optoutstudies.enabled",
 #else
 pref("app.shield.optoutstudies.enabled", false);
 #endif
 
 // Multi-lingual preferences
 pref("intl.multilingual.enabled", false);
 
 // Prio preferences
+// Only enable by default on Nightly.
+// On platforms that do not build libprio, do not set these prefs at all, which gives us a way to detect support.
+
 // Curve25519 public keys for Prio servers
+#ifdef MOZ_LIBPRIO
 pref("prio.publicKeyA", "35AC1C7576C7C6EDD7FED6BCFC337B34D48CB4EE45C86BEEFB40BD8875707733");
 pref("prio.publicKeyB", "26E6674E65425B823F1F1D5F96E3BB3EF9E406EC7FBA7DEF8B08A35DD135AF50");
+#endif
+
+// Whether or not Prio-encoded Telemetry will be sent along with the main ping.
+#if defined(NIGHTLY_BUILD) && defined(MOZ_LIBPRIO)
+pref("prio.enabled", true);
+#endif
 
 #ifdef NIGHTLY_BUILD
 pref("browser.fastblock.enabled", true);
 #endif
 
--- a/browser/moz.build
+++ b/browser/moz.build
@@ -40,16 +40,19 @@ export('DIST_SUBDIR')
 DEFINES['APP_VERSION'] = CONFIG['MOZ_APP_VERSION']
 
 for cdm in CONFIG['MOZ_EME_MODULES']:
     DEFINES['MOZ_%s_EME' % cdm.upper()] = True
 
 if CONFIG['MOZ_GPSD']:
     DEFINES['MOZ_GPSD'] = True
 
+if CONFIG['MOZ_LIBPRIO']:
+    DEFINES['MOZ_LIBPRIO'] = True
+
 # These files are specified in this moz.build to pick up DIST_SUBDIR as set in
 # this directory, which is un-set in browser/app.
 JS_PREFERENCE_PP_FILES += [
     'app/profile/firefox.js',
 ]
 FINAL_TARGET_FILES += ['app/blocklist.xml']
 FINAL_TARGET_FILES.defaults += ['app/permissions']
 
--- a/dom/chrome-webidl/PrioEncoder.webidl
+++ b/dom/chrome-webidl/PrioEncoder.webidl
@@ -2,21 +2,21 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
 [ChromeOnly, Exposed=(Window,System)]
 namespace PrioEncoder {
   [Throws, NewObject]
-  Promise<PrioEncodedData> encode(ByteString batchID, PrioParams params);
+  PrioEncodedData encode(ByteString batchID, PrioParams params);
 };
 
 dictionary PrioParams {
-  required boolean startupCrashDetected;
-  required boolean safeModeUsage;
   required boolean browserIsUserDefault;
+  required boolean newTabPageEnabled;
+  required boolean pdfViewerUsed;
 };
 
 dictionary PrioEncodedData {
   Uint8Array a;
   Uint8Array b;
 };
\ No newline at end of file
--- a/dom/prio/PrioEncoder.cpp
+++ b/dom/prio/PrioEncoder.cpp
@@ -1,19 +1,22 @@
 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "mozilla/ClearOnShutdown.h"
-#include "mozilla/dom/Promise.h"
+#include "mozilla/Preferences.h"
+#include "mozilla/ScopeExit.h"
 #include "mozilla/Services.h"
 #include "mozilla/TextUtils.h"
 
+#include "mozilla/dom/ToJSValue.h"
+
 #include "PrioEncoder.h"
 
 namespace mozilla {
 namespace dom {
 
 /* static */ StaticRefPtr<PrioEncoder> PrioEncoder::sSingleton;
 
 /* static */ PublicKey PrioEncoder::sPublicKeyA = nullptr;
@@ -30,139 +33,153 @@ PrioEncoder::~PrioEncoder()
   if (sPublicKeyB) {
     PublicKey_clear(sPublicKeyB);
     sPublicKeyB = nullptr;
   }
 
   Prio_clear();
 }
 
-/* static */
-already_AddRefed<Promise>
-PrioEncoder::Encode(GlobalObject& aGlobal, const nsCString& aBatchID, const PrioParams& aPrioParams, ErrorResult& aRv)
+/* static */ void
+PrioEncoder::Encode(GlobalObject& aGlobal,
+                    const nsCString& aBatchID,
+                    const PrioParams& aPrioParams,
+                    RootedDictionary<PrioEncodedData>& aData,
+                    ErrorResult& aRv)
 {
   nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
-
   if (!global) {
     aRv.Throw(NS_ERROR_UNEXPECTED);
-    return nullptr;
+    return;
   }
 
   SECStatus prio_rv = SECSuccess;
 
   if (!sSingleton) {
     sSingleton = new PrioEncoder();
 
     ClearOnShutdown(&sSingleton);
 
     Prio_init();
 
+    nsresult rv;
+
     nsAutoCStringN<CURVE25519_KEY_LEN_HEX + 1> prioKeyA;
-    nsresult rv = Preferences::GetCString("prio.publicKeyA", prioKeyA);
+    rv = Preferences::GetCString("prio.publicKeyA", prioKeyA);
     if (NS_FAILED(rv)) {
       aRv.Throw(NS_ERROR_UNEXPECTED);
-      return nullptr;
+      return;
     }
 
     nsAutoCStringN<CURVE25519_KEY_LEN_HEX + 1> prioKeyB;
     rv = Preferences::GetCString("prio.publicKeyB", prioKeyB);
     if (NS_FAILED(rv)) {
       aRv.Throw(NS_ERROR_UNEXPECTED);
-      return nullptr;
+      return;
     }
 
     // Check that both public keys are of the right length
     // and contain only hex digits 0-9a-fA-f
     if (!PrioEncoder::IsValidHexPublicKey(prioKeyA)
         || !PrioEncoder::IsValidHexPublicKey(prioKeyB))  {
       aRv.Throw(NS_ERROR_UNEXPECTED);
-      return nullptr;
+      return;
     }
 
-    prio_rv = PublicKey_import_hex(&sPublicKeyA, reinterpret_cast<const unsigned char*>(prioKeyA.BeginReading()), CURVE25519_KEY_LEN_HEX);
+    prio_rv = PublicKey_import_hex(&sPublicKeyA,
+                                   reinterpret_cast<const unsigned char*>(prioKeyA.BeginReading()),
+                                   CURVE25519_KEY_LEN_HEX);
     if (prio_rv != SECSuccess) {
       aRv.Throw(NS_ERROR_UNEXPECTED);
-      return nullptr;
+      return;
     }
 
-    prio_rv = PublicKey_import_hex(&sPublicKeyB, reinterpret_cast<const unsigned char*>(prioKeyB.BeginReading()), CURVE25519_KEY_LEN_HEX);
+    prio_rv = PublicKey_import_hex(&sPublicKeyB,
+              reinterpret_cast<const unsigned char*>(prioKeyB.BeginReading()),
+              CURVE25519_KEY_LEN_HEX);
     if (prio_rv != SECSuccess) {
       aRv.Throw(NS_ERROR_UNEXPECTED);
-      return nullptr;
+      return;
     }
   }
 
-  RefPtr<Promise> promise = Promise::Create(global, aRv);
-
   bool dataItems[] = {
-    aPrioParams.mStartupCrashDetected,
-    aPrioParams.mSafeModeUsage,
-    aPrioParams.mBrowserIsUserDefault
+    aPrioParams.mBrowserIsUserDefault,
+    aPrioParams.mNewTabPageEnabled,
+    aPrioParams.mPdfViewerUsed,
   };
 
-  PrioConfig prioConfig = PrioConfig_new(mozilla::ArrayLength(dataItems), sPublicKeyA, sPublicKeyB, reinterpret_cast<const unsigned char*>(aBatchID.BeginReading()), aBatchID.Length());
+  PrioConfig prioConfig = PrioConfig_new(mozilla::ArrayLength(dataItems),
+                                         sPublicKeyA,
+                                         sPublicKeyB,
+                                         reinterpret_cast<const unsigned char*>(aBatchID.BeginReading()),
+                                         aBatchID.Length());
 
   if (!prioConfig) {
-    promise->MaybeReject(NS_ERROR_FAILURE);
-    return promise.forget();
+    aRv.Throw(NS_ERROR_FAILURE);
+    return;
   }
 
   auto configGuard = MakeScopeExit([&] {
     PrioConfig_clear(prioConfig);
   });
 
   unsigned char* forServerA = nullptr;
   unsigned int lenA = 0;
   unsigned char* forServerB = nullptr;
   unsigned int lenB = 0;
 
-  prio_rv = PrioClient_encode(prioConfig, dataItems, &forServerA, &lenA, &forServerB, &lenB);
-
-  // Package the data into the dictionary
-  PrioEncodedData data;
+  prio_rv = PrioClient_encode(prioConfig,
+                              dataItems,
+                              &forServerA,
+                              &lenA,
+                              &forServerB,
+                              &lenB);
 
   nsTArray<uint8_t> arrayForServerA;
   nsTArray<uint8_t> arrayForServerB;
 
-  if (!arrayForServerA.AppendElements(reinterpret_cast<uint8_t*>(forServerA), lenA, fallible)) {
-    promise->MaybeReject(NS_ERROR_OUT_OF_MEMORY);
-    return promise.forget();
+  if (!arrayForServerA.AppendElements(reinterpret_cast<uint8_t*>(forServerA),
+                                      lenA,
+                                      fallible)) {
+    aRv.Throw(NS_ERROR_OUT_OF_MEMORY);
+    return;
   }
 
   free(forServerA);
 
-  if (!arrayForServerB.AppendElements(reinterpret_cast<uint8_t*>(forServerB), lenB, fallible)) {
-    promise->MaybeReject(NS_ERROR_OUT_OF_MEMORY);
-    return promise.forget();
+  if (!arrayForServerB.AppendElements(reinterpret_cast<uint8_t*>(forServerB),
+                                      lenB,
+                                      fallible)) {
+    aRv.Throw(NS_ERROR_OUT_OF_MEMORY);
+    return ;
   }
 
   free(forServerB);
 
+  if (prio_rv != SECSuccess) {
+    aRv.Throw(NS_ERROR_FAILURE);
+    return;
+  }
+
   JS::Rooted<JS::Value> valueA(aGlobal.Context());
   if (!ToJSValue(aGlobal.Context(), TypedArrayCreator<Uint8Array>(arrayForServerA), &valueA)) {
-    promise->MaybeReject(NS_ERROR_OUT_OF_MEMORY);
-    return promise.forget();
+    aRv.Throw(NS_ERROR_OUT_OF_MEMORY);
+    return;
   }
-  data.mA.Construct().Init(&valueA.toObject());
+
+  aData.mA.Construct().Init(&valueA.toObject());
 
   JS::Rooted<JS::Value> valueB(aGlobal.Context());
   if (!ToJSValue(aGlobal.Context(), TypedArrayCreator<Uint8Array>(arrayForServerB), &valueB)) {
-    promise->MaybeReject(NS_ERROR_OUT_OF_MEMORY);
-    return promise.forget();
-  }
-  data.mB.Construct().Init(&valueB.toObject());
-
-  if (prio_rv != SECSuccess) {
-    promise->MaybeReject(NS_ERROR_FAILURE);
-    return promise.forget();
+    aRv.Throw(NS_ERROR_OUT_OF_MEMORY);
+    return;
   }
 
-  promise->MaybeResolve(data);
-
-  return promise.forget();
+  aData.mB.Construct().Init(&valueB.toObject());
 }
 
 bool
 PrioEncoder::IsValidHexPublicKey(mozilla::Span<const char> aStr)
 {
   if (aStr.Length() != CURVE25519_KEY_LEN_HEX) {
     return false;
   }
--- a/dom/prio/PrioEncoder.h
+++ b/dom/prio/PrioEncoder.h
@@ -16,18 +16,22 @@ class nsIGlobalObject;
 namespace mozilla {
 namespace dom {
 
 class PrioEncoder
 {
 public:
   NS_INLINE_DECL_REFCOUNTING(PrioEncoder)
 
-  static already_AddRefed<Promise>
-  Encode(GlobalObject& aGlobal, const nsCString& aBatchID, const PrioParams& aPrioParams, ErrorResult& aRv);
+  static
+  void Encode(GlobalObject& aGlobal,
+              const nsCString& aBatchID,
+              const PrioParams& aPrioParams,
+              RootedDictionary<PrioEncodedData>& aData,
+              ErrorResult& aRv);
 
 private:
   PrioEncoder();
   ~PrioEncoder();
 
   static PublicKey
   sPublicKeyA;
 
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -5947,8 +5947,13 @@ pref("mozilla.widget.use-argb-visuals", 
 #endif
 
 #ifdef NIGHTLY_BUILD
 // Disable moz* APIs in DataTransfer
 pref("dom.datatransfer.mozAtAPIs", false);
 #else
 pref("dom.datatransfer.mozAtAPIs", true);
 #endif
+
+// Whether or not Prio is supported on this platform.
+#ifdef MOZ_LIBPRIO
+pref("prio.enabled", false);
+#endif
--- a/toolkit/components/telemetry/docs/data/main-ping.rst
+++ b/toolkit/components/telemetry/docs/data/main-ping.rst
@@ -672,16 +672,34 @@ Structure:
         "name": "awesomescreen.1",
         "reason": "commit",
         "start": 123,
         "end": 456
       }
       ...
     ],
 
+Prio
+----
+This section contains experimental data encoded with a basic version of the Prio system for private aggregation.
+See `the Prio paper <https://crypto.stanford.edu/prio/>`_ and `the libprio Github repo <https://github.com/mozilla/libprio>`_
+for more information.
+
+Prio splits data packets into two "shares", signed for different servers that will do the decryption+decoding and
+aggregation. We call these "Server A" and "Server B", represented as `a` and `b` keys in `payload.prio`.
+
+Structure:
+
+.. code-block:: js
+    "prio": {
+      a: ... // Uint8Array containing data signed for Server A
+      b: ... // Uint8Array containing data signed for Server B
+    }
+
+
 Version History
 ===============
 
 - Firefox 61:
 
   - Stopped reporting ``childPayloads`` (`bug 1443599 <https://bugzilla.mozilla.org/show_bug.cgi?id=1443599>`_).
   - Stopped reporting ``saved-session`` pings on Firefox Desktop (`bug 1443603 <https://bugzilla.mozilla.org/show_bug.cgi?id=1443603>`_).
   - Stopped reporting ``simpleMeasurements.js`` (`bug 1278920 <https://bugzilla.mozilla.org/show_bug.cgi?id=1278920>`_).
--- a/toolkit/components/telemetry/pings/TelemetrySession.jsm
+++ b/toolkit/components/telemetry/pings/TelemetrySession.jsm
@@ -82,16 +82,19 @@ const IDLE_TIMEOUT_SECONDS = Services.pr
 // in case of aborted sessions (currently 5 minutes).
 const ABORTED_SESSION_UPDATE_INTERVAL_MS = 5 * 60 * 1000;
 
 const TOPIC_CYCLE_COLLECTOR_BEGIN = "cycle-collector-begin";
 
 // How long to wait in millis for all the child memory reports to come in
 const TOTAL_MEMORY_COLLECTOR_TIMEOUT = 200;
 
+// Control whether Telemetry data should be encrypted with Prio.
+const PRIO_ENABLED_PREF = "prio.enabled";
+
 var gLastMemoryPoll = null;
 
 var gWasDebuggerAttached = false;
 
 XPCOMUtils.defineLazyServiceGetters(this, {
   Telemetry: ["@mozilla.org/base/telemetry;1", "nsITelemetry"],
   idleService: ["@mozilla.org/widget/idleservice;1", "nsIIdleService"],
 });
@@ -1163,16 +1166,21 @@ var Impl = {
       }
 
       // Add process measurements to payload.
       payloadObj.processes[processType] = processPayload;
     }
 
     payloadObj.info = info;
 
+    // Collect Prio-encoded measurements.
+    if (Services.prefs.getBoolPref(PRIO_ENABLED_PREF, false)) {
+      payloadObj.prio = protect(() => this._prioEncode());
+    }
+
     // Add extended set measurements for chrome process.
     if (Telemetry.canRecordExtended) {
       payloadObj.slowSQL = protect(() => Telemetry.slowSQL);
       payloadObj.fileIOReports = protect(() => Telemetry.fileIOReports);
       payloadObj.lateWrites = protect(() => Telemetry.lateWrites);
 
       payloadObj.addonDetails = protect(() => AddonManagerPrivate.getTelemetryDetails());
 
@@ -1942,9 +1950,49 @@ var Impl = {
     return TelemetryController.saveAbortedSessionPing(payload);
   },
 
   async markNewProfilePingSent() {
     this._log.trace("markNewProfilePingSent");
     this._newProfilePingSent = true;
     return TelemetryStorage.saveSessionData(this._getSessionDataObject());
   },
+
+  /**
+   * Encodes data for experimental Prio pilot project.
+   *
+   * @return {Object} An object containing Prio-encoded data.
+   */
+  _prioEncode() {
+    // First, map the Telemetry histogram names to the params PrioEncoder.encode() expects.
+    const prioEncodedHistograms = {
+      "BROWSER_IS_USER_DEFAULT": "browserIsUserDefault",
+      "NEWTAB_PAGE_ENABLED": "newTabPageEnabled",
+      "PDF_VIEWER_USED": "pdfViewerUsed",
+    };
+
+    // Build list of Prio parameters, using the first value recorded in each histogram.
+    let prioParams = {};
+    for (const [histogramName, prioName] of Object.entries(prioEncodedHistograms)) {
+      try {
+        const histogram = Telemetry.getHistogramById(histogramName);
+        const firstCount = Boolean(histogram.snapshot().sum);
+        prioParams[prioName] = firstCount;
+
+      } catch (ex) {
+        this._log.error(ex);
+      }
+    }
+
+    // Prio encode the data and add to payload.
+    const batchID = Policy.now();
+
+    let prioEncodedData;
+
+    try {
+      prioEncodedData = PrioEncoder.encode(batchID, prioParams);
+    } catch (ex) {
+      this._log.error(ex);
+    }
+
+    return prioEncodedData;
+  },
 };
--- a/toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
@@ -398,16 +398,21 @@ function checkPayload(payload, reason, s
       values: {0: 1, 1: 0},
       sum: 1,
     },
   };
   Assert.deepEqual(expected_keyed_count, keyedHistograms[TELEMETRY_TEST_KEYED_COUNT]);
 
   Assert.ok("processes" in payload, "The payload must have a processes section.");
   Assert.ok("parent" in payload.processes, "There must be at least a parent process.");
+
+  if (Services.prefs.getBoolPref("prio.enabled", false)) {
+    Assert.ok("prio" in payload, "The payload must have a prio section.");
+  }
+
   checkScalars(payload.processes);
 }
 
 function writeStringToFile(file, contents) {
   let ostream = Cc["@mozilla.org/network/safe-file-output-stream;1"]
                 .createInstance(Ci.nsIFileOutputStream);
   ostream.init(file, PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE,
                RW_OWNER, ostream.DEFER_OPEN);
--- a/toolkit/moz.configure
+++ b/toolkit/moz.configure
@@ -1065,17 +1065,19 @@ def launcher(value, target):
     if enabled:
         return True
 
 set_config('MOZ_LAUNCHER_PROCESS', launcher)
 set_define('MOZ_LAUNCHER_PROCESS', launcher)
 
 # Prio
 # ==============================================================
-@depends(c_compiler)
-def libprio(info):
+@depends(c_compiler, target)
+def libprio(info, target):
     if info:
-      if info.type in ('msvc',):
+      # TODO - re-enable Windows when bug 1489691 is fixed.
+      # Note that we will probably never support MSVC however.
+      if info.type in ('msvc',) or target.os in ('WINNT',):
         return None
     return True
 
 set_config('MOZ_LIBPRIO', libprio)
 
--- a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js
+++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js
@@ -50,16 +50,17 @@ module.exports = {
     "KeyEvent": false,
     "MatchGlob": false,
     "MatchPattern": false,
     "MatchPatternSet": false,
     "MenuBoxObject": false,
     // Specific to Firefox (Chrome code only).
     "PlacesObservers": false,
     "PlacesWeakCallbackWrapper": false,
+    "PrioEncoder": false,
     // Specific to Firefox (Chrome code only).
     "SharedArrayBuffer": false,
     "SimpleGestureEvent": false,
     // Note: StopIteration will likely be removed as part of removing legacy
     // generators, see bug 968038.
     "StopIteration": false,
     "StructuredCloneHolder": false,
     "WebAssembly": false,