Bug 1238906 - part1 : check whether audio data is audible. r=jwwang
authorAlastor Wu <alwu@mozilla.com>
Thu, 21 Jan 2016 10:19:19 +0800
changeset 280834 5c5cdbfeb6a35c7b6b7653aca89e5cdb038ace7f
parent 280833 ae02da0ca244a38b9a88f85da4c3476675d92620
child 280835 4d17eb99351084b2718a468bd8859757b01013f3
push id70606
push usercbook@mozilla.com
push dateThu, 21 Jan 2016 08:00:07 +0000
treeherdermozilla-inbound@4d17eb993510 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjwwang
bugs1238906
milestone46.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1238906 - part1 : check whether audio data is audible. r=jwwang
dom/media/MediaData.cpp
dom/media/MediaData.h
dom/media/MediaDecoderStateMachine.cpp
dom/media/MediaDecoderStateMachine.h
--- a/dom/media/MediaData.cpp
+++ b/dom/media/MediaData.cpp
@@ -49,16 +49,33 @@ AudioData::SizeOfIncludingThis(MallocSiz
 {
   size_t size = aMallocSizeOf(this) + aMallocSizeOf(mAudioData.get());
   if (mAudioBuffer) {
     size += mAudioBuffer->SizeOfIncludingThis(aMallocSizeOf);
   }
   return size;
 }
 
+bool
+AudioData::IsAudible() const
+{
+  if (!mAudioData) {
+    return false;
+  }
+
+  for (uint32_t frame = 0; frame < mFrames; ++frame) {
+    for (uint32_t channel = 0; channel < mChannels; ++channel) {
+      if (mAudioData[frame * mChannels + channel] != 0) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 /* static */
 already_AddRefed<AudioData>
 AudioData::TransferAndUpdateTimestampAndDuration(AudioData* aOther,
                                                   int64_t aTimestamp,
                                                   int64_t aDuration)
 {
   NS_ENSURE_TRUE(aOther, nullptr);
   RefPtr<AudioData> v = new AudioData(aOther->mOffset,
--- a/dom/media/MediaData.h
+++ b/dom/media/MediaData.h
@@ -144,16 +144,20 @@ public:
                                         int64_t aTimestamp,
                                         int64_t aDuration);
 
   size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
 
   // If mAudioBuffer is null, creates it from mAudioData.
   void EnsureAudioBuffer();
 
+  // To check whether mAudioData has audible signal, it's used to distinguish
+  // the audiable data and silent data.
+  bool IsAudible() const;
+
   const uint32_t mChannels;
   const uint32_t mRate;
   // At least one of mAudioBuffer/mAudioData must be non-null.
   // mChannels channels, each with mFrames frames
   RefPtr<SharedBuffer> mAudioBuffer;
   // mFrames frames, each with mChannels values
   UniquePtr<AudioDataValue[]> mAudioData;
 
--- a/dom/media/MediaDecoderStateMachine.cpp
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -111,16 +111,21 @@ static const int32_t LOW_VIDEO_THRESHOLD
 static const int AUDIO_DURATION_USECS = 40000;
 
 // If we increase our "low audio threshold" (see LOW_AUDIO_USECS above), we
 // use this as a factor in all our calculations. Increasing this will cause
 // us to be more likely to increase our low audio threshold, and to
 // increase it by more.
 static const int THRESHOLD_FACTOR = 2;
 
+// When the continuous silent data is over this threshold, means the a/v does
+// not produce any sound. This time is decided by UX suggestion, see
+// https://bugzilla.mozilla.org/show_bug.cgi?id=1235612#c18
+static const uint32_t SILENT_DATA_THRESHOLD_USECS = 10000000;
+
 namespace detail {
 
 // If we have less than this much undecoded data available, we'll consider
 // ourselves to be running low on undecoded data. We determine how much
 // undecoded data we have remaining using the reader's GetBuffered()
 // implementation.
 static const int64_t LOW_DATA_THRESHOLD_USECS = 5000000;
 
@@ -231,16 +236,18 @@ MediaDecoderStateMachine::MediaDecoderSt
   mCorruptFrames(60),
   mDecodingFirstFrame(true),
   mSentLoadedMetadataEvent(false),
   mSentFirstFrameLoadedEvent(false),
   mSentPlaybackEndedEvent(false),
   mOutputStreamManager(new OutputStreamManager()),
   mResource(aDecoder->GetResource()),
   mAudioOffloading(false),
+  mIsAudioDataAudible(false),
+  mSilentDataDuration(0),
   mBuffered(mTaskQueue, TimeIntervals(),
             "MediaDecoderStateMachine::mBuffered (Mirror)"),
   mEstimatedDuration(mTaskQueue, NullableTimeUnit(),
                     "MediaDecoderStateMachine::mEstimatedDuration (Mirror)"),
   mExplicitDuration(mTaskQueue, Maybe<double>(),
                     "MediaDecoderStateMachine::mExplicitDuration (Mirror)"),
   mPlayState(mTaskQueue, MediaDecoder::PLAY_STATE_LOADING,
              "MediaDecoderStateMachine::mPlayState (Mirror)"),
@@ -705,23 +712,45 @@ MediaDecoderStateMachine::PushFront(Medi
     VideoQueue().PushFront(aSample);
   } else {
     // TODO: Handle MediaRawData, determine which queue should be pushed.
   }
   UpdateNextFrameStatus();
 }
 
 void
+MediaDecoderStateMachine::CheckIsAudible(const MediaData* aSample)
+{
+  MOZ_ASSERT(OnTaskQueue());
+  MOZ_ASSERT(aSample->mType == MediaData::AUDIO_DATA);
+
+  const AudioData* data = aSample->As<AudioData>();
+  bool isAudible = data->IsAudible();
+  if (isAudible && !mIsAudioDataAudible) {
+    mIsAudioDataAudible = true;
+    mSilentDataDuration = 0;
+  } else if (isAudible && mIsAudioDataAudible) {
+    mSilentDataDuration += data->mDuration;
+    if (mSilentDataDuration > SILENT_DATA_THRESHOLD_USECS) {
+      mIsAudioDataAudible = false;
+      mSilentDataDuration = 0;
+    }
+  }
+}
+
+void
 MediaDecoderStateMachine::OnAudioPopped(const RefPtr<MediaData>& aSample)
 {
   MOZ_ASSERT(OnTaskQueue());
+
   mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
   UpdateNextFrameStatus();
   DispatchAudioDecodeTaskIfNeeded();
   MaybeStartBuffering();
+  CheckIsAudible(aSample);
 }
 
 void
 MediaDecoderStateMachine::OnVideoPopped(const RefPtr<MediaData>& aSample)
 {
   MOZ_ASSERT(OnTaskQueue());
   mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
   UpdateNextFrameStatus();
--- a/dom/media/MediaDecoderStateMachine.h
+++ b/dom/media/MediaDecoderStateMachine.h
@@ -400,16 +400,17 @@ protected:
   // aSample must not be null.
 
   void Push(MediaData* aSample, MediaData::Type aSampleType);
   void PushFront(MediaData* aSample, MediaData::Type aSampleType);
 
   void OnAudioPopped(const RefPtr<MediaData>& aSample);
   void OnVideoPopped(const RefPtr<MediaData>& aSample);
 
+  void CheckIsAudible(const MediaData* aSample);
   void VolumeChanged();
   void LogicalPlaybackRateChanged();
   void PreservesPitchChanged();
 
   MediaQueue<MediaData>& AudioQueue() { return mAudioQueue; }
   MediaQueue<MediaData>& VideoQueue() { return mVideoQueue; }
 
   // True if our buffers of decoded audio are not full, and we should
@@ -1190,16 +1191,20 @@ private:
 
   MediaEventProducer<MediaEventType> mOnPlaybackEvent;
   MediaEventProducer<MediaDecoderEventVisibility> mOnSeekingStart;
 
   // True if audio is offloading.
   // Playback will not start when audio is offloading.
   bool mAudioOffloading;
 
+  // Used to distiguish continuous silent audio data.
+  bool mIsAudioDataAudible;
+  uint32_t mSilentDataDuration;
+
 #ifdef MOZ_EME
   void OnCDMProxyReady(RefPtr<CDMProxy> aProxy);
   void OnCDMProxyNotReady();
   RefPtr<CDMProxy> mCDMProxy;
   MozPromiseRequestHolder<MediaDecoder::CDMProxyPromise> mCDMProxyPromise;
 #endif
 
 private: