Bug 1539262 - Count the number of prioDatas needed to encode the recorded Origin Telemetry. r=janerik
authorChris H-C <chutten@mozilla.com>
Tue, 02 Apr 2019 16:58:17 +0000
changeset 467636 c15f491f127a206fbd88c4937515ff11fe7484f9
parent 467635 41f4b3d8de33b908948fc3ae38c6630511fe823a
child 467637 47eadc1c796b782aa326f31f0b319bbf0f8afca1
push id35806
push userrgurzau@mozilla.com
push dateWed, 03 Apr 2019 04:07:39 +0000
treeherdermozilla-central@45808ab18609 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjanerik
bugs1539262
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1539262 - Count the number of prioDatas needed to encode the recorded Origin Telemetry. r=janerik In order to notify the "prio" ping when we reach the data limit, we need to keep an accounting of how many prioData elements we'd need to encode what's in storage. This also adds the pref reading and topic notification code for the "origin-telemetry-storage-limit-reached" topic that the "prio" ping observes. Differential Revision: https://phabricator.services.mozilla.com/D25127
toolkit/components/telemetry/core/TelemetryOrigin.cpp
--- a/toolkit/components/telemetry/core/TelemetryOrigin.cpp
+++ b/toolkit/components/telemetry/core/TelemetryOrigin.cpp
@@ -3,25 +3,29 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "Telemetry.h"
 #include "TelemetryOrigin.h"
 
 #include "nsDataHashtable.h"
+#include "nsIObserverService.h"
 #include "nsIXULAppInfo.h"
 #include "TelemetryCommon.h"
 #include "TelemetryOriginEnums.h"
 
 #include "mozilla/Atomics.h"
 #include "mozilla/Base64.h"
 #include "mozilla/dom/PrioEncoder.h"
+#include "mozilla/Preferences.h"
+#include "mozilla/Services.h"
 #include "mozilla/StaticMutex.h"
 
+#include <cmath>
 #include <type_traits>
 
 using mozilla::ErrorResult;
 using mozilla::MallocSizeOf;
 using mozilla::StaticMutex;
 using mozilla::StaticMutexAutoLock;
 using mozilla::dom::PrioEncoder;
 using mozilla::Telemetry::OriginMetricID;
@@ -74,16 +78,19 @@ class OriginMetricIDHashKey : public PLD
 };
 
 }  // anonymous namespace
 
 ////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////
 //
 // PRIVATE STATE, SHARED BY ALL THREADS
+//
+// Access for all of this state (except gInitDone) must be guarded by
+// gTelemetryOriginMutex.
 
 namespace {
 
 // This is a StaticMutex rather than a plain Mutex (1) so that
 // it gets initialised in a thread-safe manner the first time
 // it is used, and (2) because it is never de-initialised, and
 // a normal Mutex would show up as a leak in BloatView.  StaticMutex
 // also has the "OffTheBooks" property, so it won't show as a leak
@@ -107,16 +114,30 @@ IdToOriginsMap* gMetricToOriginsMap;
 mozilla::Atomic<bool, mozilla::Relaxed> gInitDone(false);
 
 // Useful for app-encoded data
 typedef nsDataHashtable<OriginMetricIDHashKey, nsTArray<bool>> IdToBoolsMap;
 
 static nsCString gBatchID;
 #define CANARY_BATCH_ID "decaffcoffee"
 
+// The number of prioData elements needed to encode the contents of storage.
+// Will be some whole multiple of gPrioDatasPerMetric.
+static uint32_t gPrioDataCount = 0;
+
+// Prio has a maximum supported number of bools it can encode at a time.
+// This means a single metric may result in several encoded payloads if the
+// number of origins exceeds the number of bools.
+// Each encoded payload corresponds to an element in the `prioData` array in the
+// "prio" ping.
+// This number is the number of encoded payloads needed per metric, and is
+// equivalent to "how many bitvectors do we need to encode this whole list of
+// origins?"
+static uint32_t gPrioDatasPerMetric;
+
 }  // namespace
 
 ////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////
 //
 // PRIVATE: thread-safe helpers
 
 namespace {
@@ -174,16 +195,19 @@ void TelemetryOrigin::InitializeGlobalSt
 
   // The contents and order of this array matters.
   // Both ensure a consistent app-encoding.
   gOriginsList = new nsTArray<const char*>({
       "doubleclick.de",
       "fb.com",
   });
 
+  gPrioDatasPerMetric = ceil(static_cast<double>(gOriginsList->Length()) /
+                             PrioEncoder::gNumBooleans);
+
   gOriginToIndexMap = new OriginToIndexMap(gOriginsList->Length());
   for (size_t i = 0; i < gOriginsList->Length(); ++i) {
     gOriginToIndexMap->Put(nsDependentCString((*gOriginsList)[i]), i);
   }
 
   gMetricToOriginsMap = new IdToOriginsMap();
 
   // This map shouldn't change at runtime, so make debug builds complain
@@ -244,19 +268,42 @@ nsresult TelemetryOrigin::RecordOrigin(O
   if (!gInitDone) {
     return NS_OK;
   }
 
   if (!gOriginToIndexMap->Contains(aOrigin)) {
     return NS_OK;
   }
 
+  if (!gMetricToOriginsMap->Contains(aId)) {
+    // If we haven't recorded anything for this metric yet, we're adding some
+    // prioDatas.
+    gPrioDataCount += gPrioDatasPerMetric;
+  }
+
   auto& originArray = gMetricToOriginsMap->GetOrInsert(aId);
+
+  if (originArray.Contains(aOrigin)) {
+    // If we've already recorded this metric for this origin, then we're going
+    // to need more prioDatas to encode that it happened again.
+    gPrioDataCount += gPrioDatasPerMetric;
+  }
+
   originArray.AppendElement(aOrigin);
 
+  static uint32_t sPrioPingLimit =
+      mozilla::Preferences::GetUint("toolkit.telemetry.prioping.dataLimit", 10);
+  if (gPrioDataCount >= sPrioPingLimit) {
+    nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
+    if (os) {
+      os->NotifyObservers(nullptr, "origin-telemetry-storage-limit-reached",
+                          nullptr);
+    }
+  }
+
   return NS_OK;
 }
 
 nsresult TelemetryOrigin::GetOriginSnapshot(bool aClear, JSContext* aCx,
                                             JS::MutableHandleValue aResult) {
   if (NS_WARN_IF(!XRE_IsParentProcess())) {
     return NS_ERROR_FAILURE;
   }
@@ -272,16 +319,17 @@ nsresult TelemetryOrigin::GetOriginSnaps
 
     if (aClear) {
       // I'd really prefer to clear after we're sure the snapshot didn't go
       // awry, but we can't hold a lock preventing recording while using JS
       // APIs. And replaying any interleaving recording sounds like too much
       // squeeze for not enough juice.
 
       gMetricToOriginsMap->SwapElements(copy);
+      gPrioDataCount = 0;
     } else {
       auto iter = gMetricToOriginsMap->ConstIter();
       for (; !iter.Done(); iter.Next()) {
         copy.Put(iter.Key(), iter.Data());
       }
     }
   }
 
@@ -405,16 +453,17 @@ nsresult TelemetryOrigin::GetEncodedOrig
       return NS_ERROR_FAILURE;
     }
   }
 
   // Step 4: If we need to clear, we'll need that lock again
   if (aClear) {
     StaticMutexAutoLock lock(gTelemetryOriginMutex);
     gMetricToOriginsMap->Clear();
+    gPrioDataCount = 0;
   }
 
   aSnapshot.setObject(*prioDataArray);
 
   return NS_OK;
 }
 
 /**