Bug 1457127 - Implement histogram persistence for GeckoView. r=chutten,janerik
authorAlessio Placitelli <alessio.placitelli@gmail.com>
Wed, 16 May 2018 13:09:33 +0200
changeset 419068 8fcea2ebf26e787112dc211a51a094736e06994a
parent 419067 b49beaf72b1d69e0b6e0a982d74e8f02a0f4df9d
child 419069 b07106556b62c9f9034778c7e172f4e66a744ed6
push id34025
push userapavel@mozilla.com
push dateMon, 21 May 2018 09:46:09 +0000
treeherdermozilla-central@8850728602d6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschutten, janerik
bugs1457127
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1457127 - Implement histogram persistence for GeckoView. r=chutten,janerik This patch implements the serialization and deserialization functions for plain histograms. MozReview-Commit-ID: GmnUD0xP8iI
toolkit/components/telemetry/TelemetryHistogram.cpp
toolkit/components/telemetry/TelemetryHistogram.h
toolkit/components/telemetry/geckoview/TelemetryGeckoViewPersistence.cpp
toolkit/components/telemetry/geckoview/gtest/moz.build
--- a/toolkit/components/telemetry/TelemetryHistogram.cpp
+++ b/toolkit/components/telemetry/TelemetryHistogram.cpp
@@ -13,16 +13,20 @@
 #include "nsBaseHashtable.h"
 #include "nsClassHashtable.h"
 #include "nsITelemetry.h"
 #include "nsPrintfCString.h"
 
 #include "mozilla/dom/ToJSValue.h"
 #include "mozilla/gfx/GPUProcessManager.h"
 #include "mozilla/Atomics.h"
+#if defined(MOZ_TELEMETRY_GECKOVIEW)
+// This is only used on GeckoView.
+#include "mozilla/JSONWriter.h"
+#endif
 #include "mozilla/StartupTimeline.h"
 #include "mozilla/StaticMutex.h"
 #include "mozilla/Unused.h"
 
 #include "TelemetryCommon.h"
 #include "TelemetryHistogram.h"
 #include "TelemetryScalar.h"
 #include "ipc/TelemetryIPCAccumulator.h"
@@ -31,27 +35,29 @@
 
 #include <limits>
 
 using base::Histogram;
 using base::BooleanHistogram;
 using base::CountHistogram;
 using base::FlagHistogram;
 using base::LinearHistogram;
+using mozilla::MakeTuple;
 using mozilla::StaticMutex;
 using mozilla::StaticMutexAutoLock;
 using mozilla::Telemetry::HistogramAccumulation;
 using mozilla::Telemetry::KeyedHistogramAccumulation;
 using mozilla::Telemetry::HistogramID;
 using mozilla::Telemetry::ProcessID;
 using mozilla::Telemetry::HistogramCount;
 using mozilla::Telemetry::Common::LogToBrowserConsole;
 using mozilla::Telemetry::Common::RecordedProcessType;
 using mozilla::Telemetry::Common::AutoHashtable;
 using mozilla::Telemetry::Common::GetNameForProcessID;
+using mozilla::Telemetry::Common::GetIDForProcessName;
 using mozilla::Telemetry::Common::IsExpiredVersion;
 using mozilla::Telemetry::Common::CanRecordDataset;
 using mozilla::Telemetry::Common::CanRecordProduct;
 using mozilla::Telemetry::Common::SupportedProduct;
 using mozilla::Telemetry::Common::IsInDataset;
 using mozilla::Telemetry::Common::ToJSString;
 
 namespace TelemetryIPCAccumulator = mozilla::TelemetryIPCAccumulator;
@@ -2503,8 +2509,366 @@ TelemetryHistogram::GetMapShallowSizesOf
 size_t
 TelemetryHistogram::GetHistogramSizesofIncludingThis(mozilla::MallocSizeOf
                                                      aMallocSizeOf)
 {
   StaticMutexAutoLock locker(gTelemetryHistogramMutex);
   // TODO
   return 0;
 }
+
+#if defined(MOZ_TELEMETRY_GECKOVIEW)
+namespace base {
+class PersistedSampleSet : public Histogram::SampleSet
+{
+public:
+  explicit PersistedSampleSet(const nsTArray<Histogram::Count>& aCounts,
+                              int64_t aSampleSum);
+};
+
+PersistedSampleSet::PersistedSampleSet(const nsTArray<Histogram::Count>& aCounts,
+                                       int64_t aSampleSum)
+{
+  // Initialize the data in the base class. See Histogram::SampleSet
+  // for the fields documentation.
+  const size_t numCounts = aCounts.Length();
+  counts_.resize(numCounts);
+  for (size_t i = 0; i < numCounts; i++) {
+    counts_[i] = aCounts[i];
+    redundant_count_ += aCounts[i];
+  }
+  sum_ = aSampleSum;
+};
+} // base (from ipc/chromium/src/base)
+
+namespace {
+/**
+ * Helper function to write histogram properties to JSON.
+ * Please note that this needs to be called between
+ * StartObjectProperty/EndObject calls that mark the histogram's
+ * JSON creation.
+ */
+void
+internal_ReflectHistogramToJSON(const HistogramSnapshotData& aSnapshot,
+                                mozilla::JSONWriter& aWriter)
+{
+  aWriter.IntProperty("sum", aSnapshot.mSampleSum);
+
+  // Fill the "counts" property.
+  aWriter.StartArrayProperty("counts");
+  for (size_t i = 0; i < aSnapshot.mBucketCounts.Length(); i++) {
+    aWriter.IntElement(aSnapshot.mBucketCounts[i]);
+  }
+  aWriter.EndArray();
+}
+
+bool
+internal_CanRecordHistogram(const HistogramID id,
+                            ProcessID aProcessType)
+{
+  // Check if we are allowed to record the data.
+  if (!CanRecordDataset(gHistogramInfos[id].dataset,
+                        internal_CanRecordBase(),
+                        internal_CanRecordExtended())) {
+    return false;
+  }
+
+  // Check if we're allowed to record in the given process.
+  if (aProcessType == ProcessID::Parent && !internal_IsRecordingEnabled(id)) {
+    return false;
+  }
+
+  if (aProcessType != ProcessID::Parent
+      && !CanRecordInProcess(gHistogramInfos[id].record_in_processes, aProcessType)) {
+    return false;
+  }
+
+  // Don't record if the current platform is not enabled
+  if (!CanRecordProduct(gHistogramInfos[id].products)) {
+    return false;
+  }
+
+  return true;
+}
+
+nsresult
+internal_ParseHistogramData(JSContext* aCx, JS::HandleId aEntryId,
+                            JS::HandleObject aContainerObj, nsACString& aOutName,
+                            nsTArray<Histogram::Count>& aOutCountArray, int64_t& aOutSum)
+{
+  // Get the histogram name.
+  nsAutoJSString histogramName;
+  if (!histogramName.init(aCx, aEntryId)) {
+    JS_ClearPendingException(aCx);
+    return NS_ERROR_FAILURE;
+  }
+
+  aOutName = NS_ConvertUTF16toUTF8(histogramName);
+
+  // Get the data for this histogram.
+  JS::RootedValue histogramData(aCx);
+  if (!JS_GetPropertyById(aCx, aContainerObj, aEntryId, &histogramData)) {
+    JS_ClearPendingException(aCx);
+    return NS_ERROR_FAILURE;
+  }
+
+  if (!histogramData.isObject()) {
+    // Histogram data need to be an object. If that's not the case, skip it
+    // and try to load the rest of the data.
+    return NS_ERROR_FAILURE;
+  }
+
+  // Get the "sum" property.
+  JS::RootedValue sumValue(aCx);
+  JS::RootedObject histogramObj(aCx, &histogramData.toObject());
+  if (!JS_GetProperty(aCx, histogramObj, "sum", &sumValue)) {
+    JS_ClearPendingException(aCx);
+    return NS_ERROR_FAILURE;
+  }
+
+  if (!JS::ToInt64(aCx, sumValue, &aOutSum)) {
+    JS_ClearPendingException(aCx);
+    return NS_ERROR_FAILURE;
+  }
+
+  // Get the "counts" array.
+  JS::RootedValue countsArray(aCx);
+  bool countsIsArray = false;
+  if (!JS_GetProperty(aCx, histogramObj, "counts", &countsArray)
+      || !JS_IsArrayObject(aCx, countsArray, &countsIsArray)) {
+    JS_ClearPendingException(aCx);
+    return NS_ERROR_FAILURE;
+  }
+
+  if (!countsIsArray) {
+    // The "counts" property needs to be an array. If this is not the case,
+    // skip this histogram.
+    return NS_ERROR_FAILURE;
+  }
+
+  // Get the length of the array.
+  uint32_t countsLen = 0;
+  JS::RootedObject countsArrayObj(aCx, &countsArray.toObject());
+  if (!JS_GetArrayLength(aCx, countsArrayObj, &countsLen)) {
+    JS_ClearPendingException(aCx);
+    return NS_ERROR_FAILURE;
+  }
+
+  // Parse the "counts" in the array.
+  for (uint32_t arrayIdx = 0; arrayIdx < countsLen; arrayIdx++) {
+    JS::RootedValue elementValue(aCx);
+    int countAsInt = 0;
+    if (!JS_GetElement(aCx, countsArrayObj, arrayIdx, &elementValue)
+        || !JS::ToInt32(aCx, elementValue, &countAsInt)) {
+      JS_ClearPendingException(aCx);
+      return NS_ERROR_FAILURE;
+    }
+    aOutCountArray.AppendElement(countAsInt);
+  }
+
+  return NS_OK;
+}
+
+} // Anonymous namespace
+
+nsresult
+TelemetryHistogram::SerializeHistograms(mozilla::JSONWriter& aWriter)
+{
+  MOZ_ASSERT(XRE_IsParentProcess(), "Only save histograms in the parent process");
+  if (!XRE_IsParentProcess()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  // Include the GPU process in histogram snapshots only if we actually tried
+  // to launch a process for it.
+  bool includeGPUProcess = internal_AttemptedGPUProcess();
+
+  // Take a snapshot of the histograms.
+  HistogramProcessSnapshotsArray processHistArray;
+  {
+    StaticMutexAutoLock locker(gTelemetryHistogramMutex);
+    // We always request the "opt-in"/"prerelease" dataset: we internally
+    // record the right subset, so this will only return "prerelease" if
+    // it was recorded.
+    if (NS_FAILED(internal_GetHistogramsSnapshot(locker,
+                                                 nsITelemetry::DATASET_RELEASE_CHANNEL_OPTIN,
+                                                 false /* aClearSubsession */,
+                                                 includeGPUProcess,
+                                                 processHistArray))) {
+      return NS_ERROR_FAILURE;
+    }
+  }
+
+
+
+  // Make the JSON calls on the stashed histograms for every process
+  for (uint32_t process = 0; process < processHistArray.length(); ++process) {
+    aWriter.StartObjectProperty(GetNameForProcessID(ProcessID(process)));
+
+    for (const HistogramSnapshotInfo& hData : processHistArray[process]) {
+      HistogramID id = hData.histogramID;
+
+      aWriter.StartObjectProperty(gHistogramInfos[id].name());
+      internal_ReflectHistogramToJSON(hData.data, aWriter);
+      aWriter.EndObject();
+    }
+    aWriter.EndObject();
+  }
+
+  return NS_OK;
+}
+
+nsresult
+TelemetryHistogram::DeserializeHistograms(JSContext* aCx, JS::HandleValue aData)
+{
+  MOZ_ASSERT(XRE_IsParentProcess(), "Only load histograms in the parent process");
+  if (!XRE_IsParentProcess()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  // Telemetry is disabled. This should never happen, but let's leave this check
+  // for consistency with other histogram updates routines.
+  if (!internal_CanRecordBase()) {
+    return NS_OK;
+  }
+
+  typedef mozilla::Tuple<nsCString, nsTArray<Histogram::Count>, int64_t>
+    PersistedHistogramTuple;
+  typedef mozilla::Vector<PersistedHistogramTuple> PersistedHistogramArray;
+  typedef mozilla::Vector<PersistedHistogramArray> PersistedHistogramStorage;
+
+  // Before updating the histograms, we need to get the data out of the JS
+  // wrappers. We can't hold the histogram mutex while handling JS stuff.
+  // Build a <histogram name, value> map.
+  JS::RootedObject histogramDataObj(aCx, &aData.toObject());
+  JS::Rooted<JS::IdVector> processes(aCx, JS::IdVector(aCx));
+  if (!JS_Enumerate(aCx, histogramDataObj, &processes)) {
+    // We can't even enumerate the processes in the loaded data, so
+    // there is nothing we could recover from the persistence file. Bail out.
+    JS_ClearPendingException(aCx);
+    return NS_ERROR_FAILURE;
+  }
+
+  // Make sure we have enough storage for all the processes.
+  PersistedHistogramStorage histogramsToUpdate;
+  if (!histogramsToUpdate.resize(processes.length())) {
+    return NS_ERROR_OUT_OF_MEMORY;
+  }
+
+  // The following block of code attempts to extract as much data as possible
+  // from the serialized JSON, even in case of light data corruptions: if, for example,
+  // the data for a single process is corrupted or is in an unexpected form, we press on
+  // and attempt to load the data for the other processes.
+  JS::RootedId process(aCx);
+  for (auto& processVal : processes) {
+    // This is required as JS API calls require an Handle<jsid> and not a
+    // plain jsid.
+    process = processVal;
+    // Get the process name.
+    nsAutoJSString processNameJS;
+    if (!processNameJS.init(aCx, process)) {
+      JS_ClearPendingException(aCx);
+      continue;
+    }
+
+    // Make sure it's valid. Note that this is safe to call outside
+    // of a locked section.
+    NS_ConvertUTF16toUTF8 processName(processNameJS);
+    ProcessID processID = GetIDForProcessName(processName.get());
+    if (processID == ProcessID::Count) {
+      NS_WARNING(nsPrintfCString("Failed to get process ID for %s", processName.get()).get());
+      continue;
+    }
+
+    // And its probes.
+    JS::RootedValue processData(aCx);
+    if (!JS_GetPropertyById(aCx, histogramDataObj, process, &processData)) {
+      JS_ClearPendingException(aCx);
+      continue;
+    }
+
+    if (!processData.isObject()) {
+      // |processData| should be an object containing histograms. If this is
+      // not the case, silently skip and try to load the data for the other
+      // processes.
+      continue;
+    }
+
+    // Iterate through each histogram.
+    JS::RootedObject processDataObj(aCx, &processData.toObject());
+    JS::Rooted<JS::IdVector> histograms(aCx, JS::IdVector(aCx));
+    if (!JS_Enumerate(aCx, processDataObj, &histograms)) {
+      JS_ClearPendingException(aCx);
+      continue;
+    }
+
+    // Get a reference to the deserialized data for this process.
+    PersistedHistogramArray& deserializedProcessData =
+      histogramsToUpdate[static_cast<uint32_t>(processID)];
+
+    JS::RootedId histogram(aCx);
+    for (auto& histogramVal : histograms) {
+      histogram = histogramVal;
+
+      int64_t sum = 0;
+      nsTArray<Histogram::Count> deserializedCounts;
+      nsCString histogramName;
+      if (NS_FAILED(internal_ParseHistogramData(aCx, histogram, processDataObj,
+                                                histogramName, deserializedCounts, sum))) {
+        continue;
+      }
+
+      // Finally append the deserialized data to the storage.
+      if (!deserializedProcessData.emplaceBack(
+        MakeTuple(mozilla::Move(histogramName), mozilla::Move(deserializedCounts), sum))) {
+        return NS_ERROR_OUT_OF_MEMORY;
+      }
+    }
+  }
+
+  // Update the histogram storage.
+  {
+    StaticMutexAutoLock locker(gTelemetryHistogramMutex);
+
+    for (uint32_t process = 0; process < histogramsToUpdate.length(); ++process) {
+      PersistedHistogramArray& processArray = histogramsToUpdate[process];
+
+      for (auto& histogramData : processArray) {
+        // Attempt to get the corresponding ID for the deserialized histogram name.
+        HistogramID id;
+        if (NS_FAILED(internal_GetHistogramIdByName(mozilla::Get<0>(histogramData), &id))) {
+          continue;
+        }
+
+        ProcessID procID = static_cast<ProcessID>(process);
+        if (!internal_CanRecordHistogram(id, procID)) {
+          // We're not allowed to record this, so don't try to restore it.
+          continue;
+        }
+
+        // Get the Histogram instance: this will instantiate it if it doesn't exist.
+        Histogram* h = internal_GetHistogramById(id, procID);
+        MOZ_ASSERT(h);
+
+        if (!h || internal_IsExpired(h)) {
+          // Don't restore expired histograms.
+          continue;
+        }
+
+        // Make sure that histogram counts have matching sizes. If not,
+        // |AddSampleSet| will fail and crash.
+        size_t numCounts = mozilla::Get<1>(histogramData).Length();
+        if (h->bucket_count() != numCounts) {
+          MOZ_ASSERT(false,
+                     "The number of restored buckets does not match with the on in the definition");
+          continue;
+        }
+
+        // Update the data for the histogram.
+        h->AddSampleSet(base::PersistedSampleSet(mozilla::Move(mozilla::Get<1>(histogramData)),
+                                                 mozilla::Get<2>(histogramData)));
+      }
+    }
+  }
+
+  return NS_OK;
+}
+#endif // MOZ_TELEMETRY_GECKOVIEW
--- a/toolkit/components/telemetry/TelemetryHistogram.h
+++ b/toolkit/components/telemetry/TelemetryHistogram.h
@@ -7,16 +7,23 @@
 #define TelemetryHistogram_h__
 
 #include "mozilla/TelemetryHistogramEnums.h"
 #include "mozilla/TelemetryProcessEnums.h"
 
 #include "mozilla/TelemetryComms.h"
 #include "nsXULAppAPI.h"
 
+#if defined(MOZ_TELEMETRY_GECKOVIEW)
+namespace mozilla{
+// This is only used for the GeckoView persistence.
+class JSONWriter;
+}
+#endif
+
 // This module is internal to Telemetry.  It encapsulates Telemetry's
 // histogram accumulation and storage logic.  It should only be used by
 // Telemetry.cpp.  These functions should not be used anywhere else.
 // For the public interface to Telemetry functionality, see Telemetry.h.
 
 namespace TelemetryHistogram {
 
 void InitializeGlobalState(bool canRecordBase, bool canRecordExtended);
@@ -72,11 +79,19 @@ GetKeyedHistogramSnapshots(JSContext *aC
                            bool aClearSubsession);
 
 size_t
 GetMapShallowSizesOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf);
 
 size_t
 GetHistogramSizesofIncludingThis(mozilla::MallocSizeOf aMallocSizeOf);
 
+// These functions are only meant to be used for GeckoView persistence.
+// They are responsible for updating in-memory probes with the data persisted
+// on the disk and vice-versa.
+#if defined(MOZ_TELEMETRY_GECKOVIEW)
+nsresult SerializeHistograms(mozilla::JSONWriter &aWriter);
+nsresult DeserializeHistograms(JSContext* aCx, JS::HandleValue aData);
+#endif // MOZ_TELEMETRY_GECKOVIEW
+
 } // namespace TelemetryHistogram
 
 #endif // TelemetryHistogram_h__
--- a/toolkit/components/telemetry/geckoview/TelemetryGeckoViewPersistence.cpp
+++ b/toolkit/components/telemetry/geckoview/TelemetryGeckoViewPersistence.cpp
@@ -2,16 +2,17 @@
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "TelemetryGeckoViewPersistence.h"
 
 #include "jsapi.h"
+#include "mozilla/ErrorNames.h"
 #include "mozilla/JSONWriter.h"
 #include "mozilla/Path.h"
 #include "mozilla/Preferences.h"
 #include "mozilla/ScopeExit.h"
 #include "mozilla/StaticPtr.h"
 #include "mozilla/SystemGroup.h"
 #include "mozilla/dom/ScriptSettings.h" // for AutoJSAPI
 #include "mozilla/dom/SimpleGlobalObject.h"
@@ -23,18 +24,20 @@
 #include "nsISafeOutputStream.h"
 #include "nsITimer.h"
 #include "nsLocalFile.h"
 #include "nsNetUtil.h"
 #include "nsXULAppAPI.h"
 #include "prenv.h"
 #include "prio.h"
 #include "TelemetryScalar.h"
+#include "TelemetryHistogram.h"
 #include "xpcpublic.h"
 
+using mozilla::GetErrorName;
 using mozilla::MakeScopeExit;
 using mozilla::Preferences;
 using mozilla::StaticRefPtr;
 using mozilla::SystemGroup;
 using mozilla::TaskCategory;
 using mozilla::dom::AutoJSAPI;
 using mozilla::dom::SimpleGlobalObject;
 
@@ -294,16 +297,36 @@ MainThreadParsePersistedProbes(const nsA
       ANDROID_LOG("MainThreadParsePersistedProbes - Failed to parse 'keyedScalars'.");
       MOZ_ASSERT(!JS_IsExceptionPending(jsapi.cx()), "Parsers must suppress exceptions themselves");
     }
   } else {
     // Getting the "keyedScalars" property failed, suppress the exception
     // and continue.
     JS_ClearPendingException(jsapi.cx());
   }
+
+  // Get the data for the histograms.
+  JS::RootedValue histogramData(jsapi.cx());
+  if (JS_GetProperty(jsapi.cx(), dataObj, "histograms", &histogramData)) {
+    // If the data is an object, try to parse its properties. If not,
+    // silently skip and try to load the other sections.
+    nsresult rv = NS_OK;
+    if (!histogramData.isObject()
+        || NS_FAILED(rv = TelemetryHistogram::DeserializeHistograms(jsapi.cx(), histogramData))) {
+      nsAutoCString errorName;
+      GetErrorName(rv, errorName);
+      ANDROID_LOG("MainThreadParsePersistedProbes - Failed to parse 'histograms', %s.",
+                  errorName.get());
+      MOZ_ASSERT(!JS_IsExceptionPending(jsapi.cx()), "Parsers must suppress exceptions themselves");
+    }
+  } else {
+    // Getting the "histogramData" property failed, suppress the exception
+    // and continue.
+    JS_ClearPendingException(jsapi.cx());
+  }
 }
 
 /**
  * The persistence worker function, meant to be run off the main thread.
  */
 void
 PersistenceThreadPersist()
 {
@@ -345,16 +368,22 @@ PersistenceThreadPersist()
   w.EndObject();
 
   w.StartObjectProperty("keyedScalars");
   if (NS_FAILED(TelemetryScalar::SerializeKeyedScalars(w))) {
     ANDROID_LOG("Persist - Failed to persist keyed scalars.");
   }
   w.EndObject();
 
+  w.StartObjectProperty("histograms");
+  if (NS_FAILED(TelemetryHistogram::SerializeHistograms(w))) {
+    ANDROID_LOG("Persist - Failed to persist histograms.");
+  }
+  w.EndObject();
+
   // End the building process.
   w.End();
 
   // Android can kill us while we are writing to disk and, if that happens,
   // we end up with a corrupted json overwriting the old session data.
   // Luckily, |StreamingJSONWriter::Close| is smart enough to write to a
   // temporary file and only overwrite the original file if nothing bad happened.
   nsresult rv = static_cast<StreamingJSONWriter*>(w.WriteFunc())->Close();
--- a/toolkit/components/telemetry/geckoview/gtest/moz.build
+++ b/toolkit/components/telemetry/geckoview/gtest/moz.build
@@ -17,9 +17,14 @@ LOCAL_INCLUDES += [
 
 DEFINES['MOZ_TELEMETRY_GECKOVIEW'] = True
 
 UNIFIED_SOURCES = [
     '../TelemetryGeckoViewPersistence.cpp',
     'TestGeckoView.cpp',
 ]
 
+# We need the following line otherwise including
+# "TelemetryHistogram.h" in tests will fail due to
+# missing headers.
+include('/ipc/chromium/chromium-config.mozbuild')
+
 FINAL_LIBRARY = 'xul-gtest'