Bug 1524890 - P10. Add AudioData::SetTrimWindow. r=bryce
authorJean-Yves Avenard <jyavenard@mozilla.com>
Fri, 22 Feb 2019 09:19:32 +0000
changeset 460798 08fd6ddd88e4750923b58fd8af0643a8c8d7507a
parent 460797 ddc76517c3366c8651038ad3df94ca5eb00d1058
child 460799 9b93c2718c181e8f2b7dfc0df6a0032163626300
push id35603
push userdluca@mozilla.com
push dateMon, 25 Feb 2019 01:46:40 +0000
treeherdermozilla-central@3a2ced4fbd98 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbryce
bugs1524890
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1524890 - P10. Add AudioData::SetTrimWindow. r=bryce Don't re-create a new trimmed AudioData when we want to remove some content. This remove the need for some copies. Differential Revision: https://phabricator.services.mozilla.com/D20168
dom/media/MediaData.cpp
dom/media/MediaData.h
dom/media/MediaDecoderStateMachine.cpp
dom/media/mediasink/AudioSink.cpp
--- a/dom/media/MediaData.cpp
+++ b/dom/media/MediaData.cpp
@@ -35,27 +35,83 @@ const char* VideoData::sTypeName = "vide
 bool IsDataLoudnessHearable(const AudioDataValue aData) {
   // We can transfer the digital value to dBFS via following formula. According
   // to American SMPTE standard, 0 dBu equals -20 dBFS. In theory 0 dBu is still
   // hearable, so we choose a smaller value as our threshold. If the loudness
   // is under this threshold, it might not be hearable.
   return 20.0f * std::log10(AudioSampleToFloat(aData)) > -100;
 }
 
+Span<AudioDataValue> AudioData::Data() const {
+  return MakeSpan(GetAdjustedData(), mFrames * mChannels);
+}
+
+bool AudioData::AdjustForStartTime(int64_t aStartTime) {
+  const TimeUnit startTimeOffset =
+      media::TimeUnit::FromMicroseconds(aStartTime);
+  mOriginalTime -= startTimeOffset;
+  if (mTrimWindow) {
+    *mTrimWindow -= startTimeOffset;
+  }
+  return MediaData::AdjustForStartTime(aStartTime);
+}
+
+bool AudioData::SetTrimWindow(const media::TimeInterval& aTrim) {
+  if (!mAudioData) {
+    // MoveableData got called. Can no longer work on it.
+    return false;
+  }
+  const size_t originalFrames = mAudioData.Length() / mChannels;
+  const TimeUnit originalDuration = FramesToTimeUnit(originalFrames, mRate);
+  if (aTrim.mStart < mOriginalTime ||
+      aTrim.mEnd > mOriginalTime + originalDuration) {
+    return false;
+  }
+
+  auto trimBefore = TimeUnitToFrames(aTrim.mStart - mOriginalTime, mRate);
+  auto trimAfter = aTrim.mEnd == GetEndTime()
+                       ? originalFrames
+                       : TimeUnitToFrames(aTrim.mEnd - mOriginalTime, mRate);
+  if (!trimBefore.isValid() || !trimAfter.isValid()) {
+    // Overflow.
+    return false;
+  }
+  if (!mTrimWindow && trimBefore == 0 && trimAfter == originalFrames) {
+    // Nothing to change, abort early to prevent rounding errors.
+    return true;
+  }
+
+  mTrimWindow = Some(aTrim);
+  mDataOffset = trimBefore.value() * mChannels;
+  mFrames = (trimAfter - trimBefore).value();
+  mTime = mOriginalTime + FramesToTimeUnit(trimBefore.value(), mRate);
+  mDuration = FramesToTimeUnit(mFrames, mRate);
+
+  return true;
+}
+
+AudioDataValue* AudioData::GetAdjustedData() const {
+  if (!mAudioData) {
+    return nullptr;
+  }
+  return mAudioData.Data() + mDataOffset;
+}
+
 void AudioData::EnsureAudioBuffer() {
-  if (mAudioBuffer) {
+  if (mAudioBuffer || !mAudioData) {
     return;
   }
+  const AudioDataValue* srcData = GetAdjustedData();
   mAudioBuffer =
       SharedBuffer::Create(mFrames * mChannels * sizeof(AudioDataValue));
 
-  AudioDataValue* data = static_cast<AudioDataValue*>(mAudioBuffer->Data());
+  AudioDataValue* destData = static_cast<AudioDataValue*>(mAudioBuffer->Data());
   for (uint32_t i = 0; i < mFrames; ++i) {
     for (uint32_t j = 0; j < mChannels; ++j) {
-      data[j * mFrames + i] = mAudioData[i * mChannels + j];
+      destData[j * mFrames + i] = srcData[i * mChannels + j];
     }
   }
 }
 
 size_t AudioData::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
   size_t size =
       aMallocSizeOf(this) + mAudioData.SizeOfExcludingThis(aMallocSizeOf);
   if (mAudioBuffer) {
@@ -64,26 +120,38 @@ size_t AudioData::SizeOfIncludingThis(Ma
   return size;
 }
 
 bool AudioData::IsAudible() const {
   if (!mAudioData) {
     return false;
   }
 
+  const AudioDataValue* data = GetAdjustedData();
+
   for (uint32_t frame = 0; frame < mFrames; ++frame) {
     for (uint32_t channel = 0; channel < mChannels; ++channel) {
-      if (IsDataLoudnessHearable(mAudioData[frame * mChannels + channel])) {
+      if (IsDataLoudnessHearable(data[frame * mChannels + channel])) {
         return true;
       }
     }
   }
   return false;
 }
 
+AlignedAudioBuffer AudioData::MoveableData() {
+  // Trim buffer according to trimming mask.
+  mAudioData.PopFront(mDataOffset);
+  mAudioData.SetLength(mFrames * mChannels);
+  mDataOffset = 0;
+  mFrames = 0;
+  mTrimWindow.reset();
+  return std::move(mAudioData);
+}
+
 static bool ValidatePlane(const VideoData::YCbCrBuffer::Plane& aPlane) {
   return aPlane.mWidth <= PlanarYCbCrImage::MAX_DIMENSION &&
          aPlane.mHeight <= PlanarYCbCrImage::MAX_DIMENSION &&
          aPlane.mWidth * aPlane.mHeight < MAX_VIDEO_WIDTH * MAX_VIDEO_HEIGHT &&
          aPlane.mStride > 0 && aPlane.mWidth <= aPlane.mStride;
 }
 
 static bool ValidateBufferAndPicture(const VideoData::YCbCrBuffer& aBuffer,
--- a/dom/media/MediaData.h
+++ b/dom/media/MediaData.h
@@ -291,17 +291,17 @@ class MediaData {
 
   // Duration of sample, in microseconds.
   media::TimeUnit mDuration;
 
   bool mKeyframe;
 
   media::TimeUnit GetEndTime() const { return mTime + mDuration; }
 
-  bool AdjustForStartTime(int64_t aStartTime) {
+  virtual bool AdjustForStartTime(int64_t aStartTime) {
     mTime = mTime - media::TimeUnit::FromMicroseconds(aStartTime);
     return !mTime.IsNegative();
   }
 
   template <typename ReturnType>
   const ReturnType* As() const {
     MOZ_ASSERT(this->mType == ReturnType::sType);
     return static_cast<const ReturnType*>(this);
@@ -336,63 +336,76 @@ class AudioData : public MediaData {
   AudioData(int64_t aOffset, const media::TimeUnit& aTime,
             const media::TimeUnit& aDuration, AlignedAudioBuffer&& aData,
             uint32_t aChannels, uint32_t aRate,
             uint32_t aChannelMap = AudioConfig::ChannelLayout::UNKNOWN_MAP)
       : MediaData(sType, aOffset, aTime, aDuration),
         mChannels(aChannels),
         mChannelMap(aChannelMap),
         mRate(aRate),
+        mOriginalTime(aTime),
         mAudioData(std::move(aData)),
         mFrames(mAudioData.Length() / aChannels) {}
 
   static const Type sType = Type::AUDIO_DATA;
   static const char* sTypeName;
 
   // Access the buffer as a Span.
-  Span<AudioDataValue> Data() const {
-    return MakeSpan(mAudioData.Data(), mAudioData.Length());
-  }
+  Span<AudioDataValue> Data() const;
 
   // Amount of frames for contained data.
   uint32_t Frames() const { return mFrames; }
 
+  // Trim the audio buffer such that its apparent content fits within the aTrim
+  // interval. The actual data isn't removed from the buffer and a followup call
+  // to SetTrimWindow could restore the content. mDuration, mTime and mFrames
+  // will be adjusted accordingly.
+  // Warning: rounding may occurs, in which case the new start time of the audio
+  // sample may still be lesser than aTrim.mStart.
+  bool SetTrimWindow(const media::TimeInterval& aTrim);
+
   // Get the internal audio buffer to be moved. After this call the original
   // AudioData will be emptied and can't be used again.
-  AlignedAudioBuffer MoveableData() { return std::move(mAudioData); }
+  AlignedAudioBuffer MoveableData();
 
   size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
 
   // If mAudioBuffer is null, creates it from mAudioData.
   void EnsureAudioBuffer();
 
   // To check whether mAudioData has audible signal, it's used to distinguish
   // the audiable data and silent data.
   bool IsAudible() const;
 
+  bool AdjustForStartTime(int64_t aStartTime) override;
+
   const uint32_t mChannels;
   // The AudioConfig::ChannelLayout map. Channels are ordered as per SMPTE
   // definition. A value of UNKNOWN_MAP indicates unknown layout.
   // ChannelMap is an unsigned bitmap compatible with Windows' WAVE and FFmpeg
   // channel map.
   const AudioConfig::ChannelLayout::ChannelMap mChannelMap;
   const uint32_t mRate;
 
   // At least one of mAudioBuffer/mAudioData must be non-null.
   // mChannels channels, each with mFrames frames
   RefPtr<SharedBuffer> mAudioBuffer;
 
  protected:
   ~AudioData() {}
 
  private:
+  AudioDataValue* GetAdjustedData() const;
+  media::TimeUnit mOriginalTime;
   // mFrames frames, each with mChannels values
   AlignedAudioBuffer mAudioData;
+  Maybe<media::TimeInterval> mTrimWindow;
   // Amount of frames for contained data.
-  const uint32_t mFrames;
+  uint32_t mFrames;
+  size_t mDataOffset = 0;
 };
 
 namespace layers {
 class TextureClient;
 class PlanarYCbCrImage;
 }  // namespace layers
 
 class VideoInfo;
--- a/dom/media/MediaDecoderStateMachine.cpp
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -1321,48 +1321,25 @@ class MediaDecoderStateMachine::Accurate
     // The seek target lies somewhere in this AudioData's frames, strip off
     // any frames which lie before the seek target, so we'll begin playback
     // exactly at the seek target.
     NS_ASSERTION(mSeekJob.mTarget->GetTime() >= audioTime,
                  "Target must at or be after data start.");
     NS_ASSERTION(mSeekJob.mTarget->GetTime() < audioTime + sampleDuration,
                  "Data must end after target.");
 
-    CheckedInt64 framesToPrune = TimeUnitToFrames(
-        mSeekJob.mTarget->GetTime() - audioTime, Info().mAudio.mRate);
-    if (!framesToPrune.isValid()) {
+    bool ok = aAudio->SetTrimWindow(
+        {mSeekJob.mTarget->GetTime(), aAudio->GetEndTime()});
+    if (!ok) {
       return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
     }
-    if (framesToPrune.value() > aAudio->Frames()) {
-      // We've messed up somehow. Don't try to trim frames, the |frames|
-      // variable below will overflow.
-      SLOGE("Can't prune more frames that we have!");
-      return NS_ERROR_FAILURE;
-    }
-    uint32_t frames = aAudio->Frames() - uint32_t(framesToPrune.value());
-    uint32_t channels = aAudio->mChannels;
-    AlignedAudioBuffer audioData(frames * channels);
-    if (!audioData) {
-      return NS_ERROR_OUT_OF_MEMORY;
-    }
-
-    Span<const AudioDataValue> audioDataRange = aAudio->Data();
-    memcpy(audioData.get(),
-           audioDataRange.Elements() + (framesToPrune.value() * channels),
-           frames * channels * sizeof(AudioDataValue));
-    auto duration = FramesToTimeUnit(frames, Info().mAudio.mRate);
-    if (!duration.IsValid()) {
-      return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
-    }
-    RefPtr<AudioData> data(new AudioData(
-        aAudio->mOffset, mSeekJob.mTarget->GetTime(), duration,
-        std::move(audioData), channels, aAudio->mRate, aAudio->mChannelMap));
+
     MOZ_ASSERT(AudioQueue().GetSize() == 0,
                "Should be the 1st sample after seeking");
-    mMaster->PushAudio(data);
+    mMaster->PushAudio(aAudio);
     mDoneAudioSeeking = true;
 
     return NS_OK;
   }
 
   nsresult DropVideoUpToSeekTarget(VideoData* aVideo) {
     MOZ_ASSERT(aVideo);
     SLOG("DropVideoUpToSeekTarget() frame [%" PRId64 ", %" PRId64 "]",
--- a/dom/media/mediasink/AudioSink.cpp
+++ b/dom/media/mediasink/AudioSink.cpp
@@ -411,21 +411,17 @@ void AudioSink::NotifyAudioNeeded() {
       }
       PushProcessedAudio(silenceData);
     }
 
     mLastEndTime = data->GetEndTime();
     mFramesParsed += data->Frames();
 
     if (mConverter->InputConfig() != mConverter->OutputConfig()) {
-      // We must ensure that the size in the buffer contains exactly the number
-      // of frames, in case one of the audio producer over allocated the buffer.
       AlignedAudioBuffer buffer(data->MoveableData());
-      buffer.SetLength(size_t(data->Frames()) * data->mChannels);
-
       AlignedAudioBuffer convertedData =
           mConverter->Process(AudioSampleBuffer(std::move(buffer))).Forget();
       data = CreateAudioFromBuffer(std::move(convertedData), data);
     }
     if (PushProcessedAudio(data)) {
       mLastProcessedPacket = Some(data);
     }
   }