Bug 1482259 - Add Telemetry to know the proportion of silent part in the whole audio track. r=cpearce,francois
authoralwu <alwu@mozilla.com>
Wed, 15 Aug 2018 16:35:51 +0000
changeset 486746 8fbba4053aa0032f7ea5ced9b94c1c3972deee29
parent 486745 907a277d51d8196af0f14ba3ef3f37963303814e
child 486747 92832d7ecc29acb44a81ceb7836baaf528c20282
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscpearce, francois
bugs1482259
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1482259 - Add Telemetry to know the proportion of silent part in the whole audio track. r=cpearce,francois Use new telemetry histogram ID 'AUDIO_TRACK_SILENCE_PROPORTION' to know the proportion of silent part in the whole audio track. Differential Revision: https://phabricator.services.mozilla.com/D3066
dom/html/HTMLMediaElement.cpp
dom/html/HTMLMediaElement.h
toolkit/components/telemetry/Histograms.json
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -1745,16 +1745,17 @@ HTMLMediaElement::ShutdownDecoder()
     }
     out.mNextAvailableTrackID = std::max<TrackID>(
       mDecoder->NextAvailableTrackIDFor(out.mStream->GetInputStream()),
       out.mNextAvailableTrackID);
   }
   mDecoder->Shutdown();
   DDUNLINKCHILD(mDecoder.get());
   mDecoder = nullptr;
+  ReportAudioTrackSilenceProportionTelemetry();
 }
 
 void
 HTMLMediaElement::AbortExistingLoads()
 {
   // Abort any already-running instance of the resource selection algorithm.
   mLoadWaitStatus = NOT_WAITING;
 
@@ -1822,16 +1823,17 @@ HTMLMediaElement::AbortExistingLoads()
   mDownloadSuspendedByCache = false;
   mMediaInfo = MediaInfo();
   mIsEncrypted = false;
   mPendingEncryptedInitData.Reset();
   mWaitingForKey = NOT_WAITING_FOR_KEY;
   mSourcePointer = nullptr;
 
   mTags = nullptr;
+  mAudioTrackSilenceStartedTime = 0.0;
 
   if (mNetworkState != NETWORK_EMPTY) {
     NS_ASSERTION(!mDecoder && !mSrcStream,
                  "How did someone setup a new stream/decoder already?");
     // ChangeNetworkState() will call UpdateAudioChannelPlayingState()
     // indirectly which depends on mPaused. So we need to update mPaused first.
     if (!mPaused) {
       mPaused = true;
@@ -2898,16 +2900,28 @@ HTMLMediaElement::Seek(double aTime,
   //       actual seek target before beginning the synchronous section, but
   //       that requires changing all MediaDecoderReaders to support telling
   //       us the fastSeek target, and it's currently not possible to get
   //       this information as we don't yet control the demuxer for all
   //       MediaDecoderReaders.
 
   mPlayingBeforeSeek = IsPotentiallyPlaying();
 
+  // If the audio track is silent before seeking, we should end current silence
+  // range and start a new range after seeking. Since seek() could be called
+  // multiple times before seekEnd() executed, we should only calculate silence
+  // range when first time seek() called. Calculating on other seek() calls
+  // would cause a wrong result. In order to get correct time, this checking
+  // should be called before decoder->seek().
+  if (IsAudioTrackCurrentlySilent() &&
+      !mHasAccumulatedSilenceRangeBeforeSeekEnd) {
+    AccumulateAudioTrackSilence();
+    mHasAccumulatedSilenceRangeBeforeSeekEnd = true;
+  }
+
   // The media backend is responsible for dispatching the timeupdate
   // event if it changes the playback position as a result of the seek.
   LOG(LogLevel::Debug, ("%p SetCurrentTime(%f) starting seek", this, aTime));
   mDecoder->Seek(aTime, aSeekType);
 
   // We changed whether we're seeking so we need to AddRemoveSelfReference.
   AddRemoveSelfReference();
 
@@ -5710,16 +5724,23 @@ HTMLMediaElement::SeekCompleted()
   }
   FireTimeUpdate(false);
   DispatchAsyncEvent(NS_LITERAL_STRING("seeked"));
   // We changed whether we're seeking so we need to AddRemoveSelfReference
   AddRemoveSelfReference();
   if (mCurrentPlayRangeStart == -1.0) {
     mCurrentPlayRangeStart = CurrentTime();
   }
+
+  // After seeking completed, if the audio track is silent, start another new
+  // silence range.
+  mHasAccumulatedSilenceRangeBeforeSeekEnd = false;
+  if (IsAudioTrackCurrentlySilent()) {
+    UpdateAudioTrackSilenceRange(mIsAudioTrackAudible);
+  }
 }
 
 void
 HTMLMediaElement::NotifySuspendedByCache(bool aSuspendedByCache)
 {
   mDownloadSuspendedByCache = aSuspendedByCache;
   UpdateReadyStateInternal();
 }
@@ -7579,22 +7600,93 @@ HTMLMediaElement::IsCurrentlyPlaying() c
   // current data.
   return mReadyState >= HAVE_CURRENT_DATA && !IsPlaybackEnded();
 }
 
 void
 HTMLMediaElement::SetAudibleState(bool aAudible)
 {
   if (mIsAudioTrackAudible != aAudible) {
+    UpdateAudioTrackSilenceRange(aAudible);
     mIsAudioTrackAudible = aAudible;
     NotifyAudioPlaybackChanged(
       AudioChannelService::AudibleChangedReasons::eDataAudibleChanged);
   }
 }
 
+bool
+HTMLMediaElement::IsAudioTrackCurrentlySilent() const
+{
+  return HasAudio() && !mIsAudioTrackAudible;
+}
+
+void
+HTMLMediaElement::UpdateAudioTrackSilenceRange(bool aAudible)
+{
+  if (!HasAudio()) {
+    return;
+  }
+
+  if (!aAudible) {
+    mAudioTrackSilenceStartedTime = CurrentTime();
+    return;
+  }
+
+  AccumulateAudioTrackSilence();
+}
+
+void
+HTMLMediaElement::AccumulateAudioTrackSilence()
+{
+  MOZ_ASSERT(HasAudio());
+  const double current = CurrentTime();
+  if (current < mAudioTrackSilenceStartedTime) {
+    return;
+  }
+  const auto start = media::TimeUnit::FromSeconds(mAudioTrackSilenceStartedTime);
+  const auto end = media::TimeUnit::FromSeconds(current);
+  mSilenceTimeRanges += media::TimeInterval(start, end);
+}
+
+void
+HTMLMediaElement::ReportAudioTrackSilenceProportionTelemetry()
+{
+  if (!HasAudio()) {
+    return;
+  }
+
+  // Add last silence range to our ranges set.
+  if (!mIsAudioTrackAudible) {
+    AccumulateAudioTrackSilence();
+  }
+
+  RefPtr<TimeRanges> ranges = Played();
+  const uint32_t lengthPlayedRange = ranges->Length();
+  const uint32_t lengthSilenceRange = mSilenceTimeRanges.Length();
+  if (!lengthPlayedRange || !lengthSilenceRange) {
+    return;
+  }
+
+  double playedTime = 0.0, silenceTime = 0.0;
+  for (uint32_t idx = 0; idx < lengthPlayedRange; idx++) {
+    playedTime += ranges->End(idx) - ranges->Start(idx);
+  }
+
+  for (uint32_t idx = 0; idx < lengthSilenceRange; idx++) {
+    silenceTime +=
+      mSilenceTimeRanges.End(idx).ToSeconds() - mSilenceTimeRanges.Start(idx).ToSeconds();
+  }
+
+  double silenceProportion = (silenceTime / playedTime) * 100;
+  // silenceProportion should be in the range [0, 100]
+  silenceProportion = std::min(100.0, std::max(silenceProportion, 0.0));
+  Telemetry::Accumulate(Telemetry::AUDIO_TRACK_SILENCE_PROPORTION,
+                        silenceProportion);
+}
+
 void
 HTMLMediaElement::NotifyAudioPlaybackChanged(AudibleChangedReasons aReason)
 {
   if (mAudioChannelWrapper) {
     mAudioChannelWrapper->NotifyAudioPlaybackChanged(aReason);
   }
   // only request wake lock for audible media.
   UpdateWakeLock();
--- a/dom/html/HTMLMediaElement.h
+++ b/dom/html/HTMLMediaElement.h
@@ -1364,16 +1364,33 @@ protected:
   // If the media element's tab has never been in the foreground, this
   // registers as with the AudioChannelAgent to notify us when the tab
   // is put in the foreground, whereupon we will begin playback.
   bool AudioChannelAgentDelayingPlayback();
 
   // Ensures we're prompting the user for permission to autoplay.
   void EnsureAutoplayRequested(bool aHandlingUserInput);
 
+  // Update the silence range of the audio track when the audible status of
+  // silent audio track changes or seeking to the new position where the audio
+  // track is silent.
+  void UpdateAudioTrackSilenceRange(bool aAudible);
+
+  // When silent audio track becomes audible or seeking to new place, we would
+  // end the current silence range and accumulate it to the total silence
+  // proportion of audio track and update current silence range.
+  void AccumulateAudioTrackSilence();
+
+  // True when the media element's audio track is containing silence now.
+  bool IsAudioTrackCurrentlySilent() const;
+
+  // Calculate the audio track silence proportion and then report the telemetry
+  // result. we would report the result when decoder is destroyed.
+  void ReportAudioTrackSilenceProportionTelemetry();
+
   // The current decoder. Load() has been called on this decoder.
   // At most one of mDecoder and mSrcStream can be non-null.
   RefPtr<MediaDecoder> mDecoder;
 
   // The DocGroup-specific nsISerialEventTarget of this HTML element on the main
   // thread.
   nsCOMPtr<nsISerialEventTarget> mMainThreadEventTarget;
 
@@ -1482,16 +1499,27 @@ protected:
   LoadAlgorithmState mLoadWaitStatus = NOT_WAITING;
 
   // Current audio volume
   double mVolume = 1.0;
 
   // True if the audio track is not silent.
   bool mIsAudioTrackAudible = false;
 
+  // Used to mark the start of the silence range of audio track.
+  double mAudioTrackSilenceStartedTime = 0.0;
+
+  // Save all the silence ranges, all ranges would be normalized. That means
+  // intervals won't overlap or touch each other.
+  media::TimeIntervals mSilenceTimeRanges;
+
+  // True if we have calculated silence range before SeekEnd(). This attribute
+  // would be reset after seeking completed.
+  bool mHasAccumulatedSilenceRangeBeforeSeekEnd = false;
+
   enum MutedReasons {
     MUTED_BY_CONTENT               = 0x01,
     MUTED_BY_INVALID_PLAYBACK_RATE = 0x02,
     MUTED_BY_AUDIO_CHANNEL         = 0x04,
     MUTED_BY_AUDIO_TRACK           = 0x08
   };
 
   uint32_t mMuted = 0;
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -13914,10 +13914,25 @@
     "record_in_processes": ["main", "gpu"],
     "alert_emails": ["kgupta@mozilla.com"],
     "bug_numbers": [1470901],
     "expires_in_version": "never",
     "kind": "exponential",
     "high": 1000,
     "n_buckets": 50,
     "description": "WebRender render time in milliseconds"
+  },
+  "AUDIO_TRACK_SILENCE_PROPORTION": {
+    "record_in_processes": ["main", "content"],
+    "alert_emails": [
+      "alwu@mozilla.com",
+      "cpearce@mozilla.com",
+      "nohlmeier@mozilla.com"
+    ],
+    "expires_in_version": "68",
+    "kind": "linear",
+    "high": 100,
+    "n_buckets": 10,
+    "bug_numbers": [1482259],
+    "description": "In order to know how many silent audio track in the web comparing to other normal media, we record the silence proportion of audio track of media element. The range of value is from 0% - 100%.",
+    "releaseChannelCollection": "opt-out"
   }
 }