Bug 1568058 - pt 3 - Remove MediaFormatReader from MediaBufferDecoder. r=jya
authorMichael Froman <mfroman@mozilla.com>
Thu, 14 Nov 2019 16:06:46 +0000
changeset 502087 53a35e1379d64cb80fe1df622afb9aa8020dbcf1
parent 502086 08c7f0dd63c8f09cd5c3f678920c9b89622637bc
child 502088 db32a8eb28a2d053f009765785bac23f4f9f6d3f
push id114172
push userdluca@mozilla.com
push dateTue, 19 Nov 2019 11:31:10 +0000
treeherdermozilla-inbound@b5c5ba07d3db [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjya
bugs1568058
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1568058 - pt 3 - Remove MediaFormatReader from MediaBufferDecoder. r=jya Stop using MediaFormatReader and use a demuxer and decoder directly in MediaBufferDecoder. This will allow us to do batch decoding calls for webaudio that will improve performance by reducing the number of IPC calls to the RDD process. Differential Revision: https://phabricator.services.mozilla.com/D51453
dom/media/webaudio/MediaBufferDecoder.cpp
--- a/dom/media/webaudio/MediaBufferDecoder.cpp
+++ b/dom/media/webaudio/MediaBufferDecoder.cpp
@@ -8,28 +8,29 @@
 #include "mozilla/dom/AudioContextBinding.h"
 #include "mozilla/dom/BaseAudioContextBinding.h"
 #include "mozilla/dom/DOMException.h"
 #include "mozilla/dom/ScriptSettings.h"
 #include "mozilla/AbstractThread.h"
 #include <speex/speex_resampler.h>
 #include "nsXPCOMCIDInternal.h"
 #include "nsComponentManagerUtils.h"
-#include "MediaFormatReader.h"
 #include "MediaQueue.h"
 #include "BufferMediaResource.h"
 #include "DecoderTraits.h"
 #include "AudioContext.h"
 #include "AudioBuffer.h"
 #include "js/MemoryFunctions.h"
 #include "MediaContainerType.h"
+#include "MediaDataDemuxer.h"
 #include "nsContentUtils.h"
 #include "nsIScriptObjectPrincipal.h"
 #include "nsIScriptError.h"
 #include "nsMimeTypes.h"
+#include "PDMFactory.h"
 #include "VideoUtils.h"
 #include "WebAudioUtils.h"
 #include "mozilla/dom/Promise.h"
 #include "mozilla/Telemetry.h"
 #include "nsPrintfCString.h"
 #include "AudioNodeEngine.h"
 
 namespace mozilla {
@@ -74,31 +75,28 @@ class MediaDecodeTask final : public Run
  public:
   MediaDecodeTask(const MediaContainerType& aContainerType, uint8_t* aBuffer,
                   uint32_t aLength, WebAudioDecodeJob& aDecodeJob)
       : Runnable("MediaDecodeTask"),
         mContainerType(aContainerType),
         mBuffer(aBuffer),
         mLength(aLength),
         mDecodeJob(aDecodeJob),
-        mPhase(PhaseEnum::Decode),
-        mFirstFrameDecoded(false) {
+        mPhase(PhaseEnum::Decode) {
     MOZ_ASSERT(aBuffer);
     MOZ_ASSERT(NS_IsMainThread());
   }
 
   // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.  See
   // bug 1535398.
   MOZ_CAN_RUN_SCRIPT_BOUNDARY
   NS_IMETHOD Run() override;
-  bool CreateReader();
-  MediaFormatReader* Reader() {
-    MOZ_ASSERT(mDecoderReader);
-    return mDecoderReader;
-  }
+  bool Init();
+  TaskQueue* PDecoderTaskQueue() { return mPDecoderTaskQueue; }
+  bool OnPDecoderTaskQueue() { return mPDecoderTaskQueue->IsCurrentThreadIn(); }
 
  private:
   MOZ_CAN_RUN_SCRIPT
   void ReportFailureOnMainThread(WebAudioDecodeJob::ErrorCode aErrorCode) {
     if (NS_IsMainThread()) {
       Cleanup();
       mDecodeJob.OnFailure(aErrorCode);
     } else {
@@ -108,83 +106,105 @@ class MediaDecodeTask final : public Run
 
       nsCOMPtr<nsIRunnable> event = new ReportResultTask(
           mDecodeJob, &WebAudioDecodeJob::OnFailure, aErrorCode);
       mMainThread->Dispatch(event.forget());
     }
   }
 
   void Decode();
-  MOZ_CAN_RUN_SCRIPT void OnMetadataRead(MetadataHolder&& aMetadata);
-  MOZ_CAN_RUN_SCRIPT void OnMetadataNotRead(const MediaResult& aError);
-  void RequestSample();
-  void SampleDecoded(RefPtr<AudioData> aData);
-  MOZ_CAN_RUN_SCRIPT void SampleNotDecoded(const MediaResult& aError);
+
+  MediaResult CreateDecoder(const AudioInfo& info);
+
+  MOZ_CAN_RUN_SCRIPT void OnInitDemuxerCompleted();
+  MOZ_CAN_RUN_SCRIPT void OnInitDemuxerFailed(const MediaResult& aError);
+
+  void InitDecoder();
+  void OnInitDecoderCompleted();
+  MOZ_CAN_RUN_SCRIPT void OnInitDecoderFailed();
+
+  void DoDemux();
+  void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
+  MOZ_CAN_RUN_SCRIPT void OnAudioDemuxFailed(const MediaResult& aError);
+
+  void DoDecode();
+  void OnAudioDecodeCompleted(MediaDataDecoder::DecodedData&& aResults);
+  MOZ_CAN_RUN_SCRIPT void OnAudioDecodeFailed(const MediaResult& aError);
+
+  void DoDrain();
+  MOZ_CAN_RUN_SCRIPT void OnAudioDrainCompleted(
+      MediaDataDecoder::DecodedData&& aResults);
+  MOZ_CAN_RUN_SCRIPT void OnAudioDrainFailed(const MediaResult& aError);
+
+  void ShutdownDecoder();
+
   MOZ_CAN_RUN_SCRIPT void FinishDecode();
   MOZ_CAN_RUN_SCRIPT void AllocateBuffer();
   MOZ_CAN_RUN_SCRIPT void CallbackTheResult();
 
   void Cleanup() {
     MOZ_ASSERT(NS_IsMainThread());
-    mDecoderReader = nullptr;
     JS_free(nullptr, mBuffer);
+    if (mTrackDemuxer) {
+      mTrackDemuxer->BreakCycles();
+    }
+    mTrackDemuxer = nullptr;
+    mDemuxer = nullptr;
+    mPDecoderTaskQueue = nullptr;
   }
 
  private:
   MediaContainerType mContainerType;
   uint8_t* mBuffer;
   uint32_t mLength;
   WebAudioDecodeJob& mDecodeJob;
   PhaseEnum mPhase;
-  RefPtr<MediaFormatReader> mDecoderReader;
+  RefPtr<TaskQueue> mPDecoderTaskQueue;
+  RefPtr<MediaDataDemuxer> mDemuxer;
+  RefPtr<MediaTrackDemuxer> mTrackDemuxer;
+  RefPtr<MediaDataDecoder> mDecoder;
+  nsTArray<RefPtr<MediaRawData>> mRawSamples;
   MediaInfo mMediaInfo;
   MediaQueue<AudioData> mAudioQueue;
   RefPtr<AbstractThread> mMainThread;
-  bool mFirstFrameDecoded;
 };
 
 NS_IMETHODIMP
 MediaDecodeTask::Run() {
-  MOZ_ASSERT(mDecoderReader);
   switch (mPhase) {
     case PhaseEnum::Decode:
       Decode();
       break;
     case PhaseEnum::AllocateBuffer:
       AllocateBuffer();
       break;
     case PhaseEnum::Done:
       break;
   }
 
   return NS_OK;
 }
 
-bool MediaDecodeTask::CreateReader() {
+bool MediaDecodeTask::Init() {
   MOZ_ASSERT(NS_IsMainThread());
 
   RefPtr<BufferMediaResource> resource =
       new BufferMediaResource(static_cast<uint8_t*>(mBuffer), mLength);
 
   mMainThread = mDecodeJob.mContext->GetOwnerGlobal()->AbstractMainThreadFor(
       TaskCategory::Other);
 
+  mPDecoderTaskQueue =
+      new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
+                    "MediaBufferDecoder::mPDecoderTaskQueue");
+
   // If you change this list to add support for new decoders, please consider
   // updating HTMLMediaElement::CreateDecoder as well.
-
-  MediaFormatReaderInit init;
-  init.mResource = resource;
-  mDecoderReader = DecoderTraits::CreateReader(mContainerType, init);
-
-  if (!mDecoderReader) {
-    return false;
-  }
-
-  nsresult rv = mDecoderReader->Init();
-  if (NS_FAILED(rv)) {
+  mDemuxer = DecoderTraits::CreateDemuxer(mContainerType, resource);
+  if (!mDemuxer) {
     return false;
   }
 
   return true;
 }
 
 class AutoResampler final {
  public:
@@ -201,84 +221,216 @@ class AutoResampler final {
   void operator=(SpeexResamplerState* aResampler) { mResampler = aResampler; }
 
  private:
   SpeexResamplerState* mResampler;
 };
 
 void MediaDecodeTask::Decode() {
   MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(OnPDecoderTaskQueue());
 
-  mDecoderReader->AsyncReadMetadata()->Then(
-      mDecoderReader->OwnerThread(), __func__, this,
-      &MediaDecodeTask::OnMetadataRead, &MediaDecodeTask::OnMetadataNotRead);
+  mDemuxer->Init()->Then(PDecoderTaskQueue(), __func__, this,
+                         &MediaDecodeTask::OnInitDemuxerCompleted,
+                         &MediaDecodeTask::OnInitDemuxerFailed);
 }
 
-void MediaDecodeTask::OnMetadataRead(MetadataHolder&& aMetadata) {
-  mMediaInfo = *aMetadata.mInfo;
-  if (!mMediaInfo.HasAudio()) {
-    mDecoderReader->Shutdown();
-    ReportFailureOnMainThread(WebAudioDecodeJob::NoAudio);
-    return;
+void MediaDecodeTask::OnInitDemuxerCompleted() {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  if (!!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack)) {
+    mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
+    if (!mTrackDemuxer) {
+      ReportFailureOnMainThread(WebAudioDecodeJob::UnknownContent);
+      return;
+    }
+
+    RefPtr<PDMFactory> platform = new PDMFactory();
+    UniquePtr<TrackInfo> audioInfo = mTrackDemuxer->GetInfo();
+    // We actively ignore audio tracks that we know we can't play.
+    if (audioInfo && audioInfo->IsValid() &&
+        platform->SupportsMimeType(audioInfo->mMimeType, nullptr)) {
+      mMediaInfo.mAudio = *audioInfo->GetAsAudioInfo();
+    }
   }
 
-  nsCString codec;
-  if (!mMediaInfo.mAudio.GetAsAudioInfo()->mMimeType.IsEmpty()) {
-    codec = nsPrintfCString(
-        "webaudio; %s", mMediaInfo.mAudio.GetAsAudioInfo()->mMimeType.get());
-  } else {
-    codec = nsPrintfCString("webaudio;resource; %s",
-                            mContainerType.Type().AsString().Data());
+  if (NS_FAILED(CreateDecoder(*mMediaInfo.mAudio.GetAsAudioInfo()))) {
+    ReportFailureOnMainThread(WebAudioDecodeJob::UnknownContent);
+    return;
   }
-
-  nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
-      "MediaDecodeTask::OnMetadataRead", [codec]() -> void {
-        MOZ_ASSERT(!codec.IsEmpty());
-        MOZ_LOG(gMediaDecoderLog, LogLevel::Debug,
-                ("Telemetry (WebAudio) MEDIA_CODEC_USED= '%s'", codec.get()));
-        Telemetry::Accumulate(Telemetry::HistogramID::MEDIA_CODEC_USED, codec);
-      });
-  SystemGroup::Dispatch(TaskCategory::Other, task.forget());
-
-  RequestSample();
+  InitDecoder();
 }
 
-void MediaDecodeTask::OnMetadataNotRead(const MediaResult& aReason) {
-  mDecoderReader->Shutdown();
+void MediaDecodeTask::OnInitDemuxerFailed(const MediaResult& aError) {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
   ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
 }
 
-void MediaDecodeTask::RequestSample() {
-  mDecoderReader->RequestAudioData()->Then(
-      mDecoderReader->OwnerThread(), __func__, this,
-      &MediaDecodeTask::SampleDecoded, &MediaDecodeTask::SampleNotDecoded);
+MediaResult MediaDecodeTask::CreateDecoder(const AudioInfo& info) {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  RefPtr<PDMFactory> pdm = new PDMFactory();
+  // result may not be updated by PDMFactory::CreateDecoder, as such it must be
+  // initialized to a fatal error by default.
+  MediaResult result =
+      MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+                  nsPrintfCString("error creating %s decoder",
+                                  TrackTypeToStr(TrackInfo::kAudioTrack)));
+  mDecoder = pdm->CreateDecoder(
+      {info, mPDecoderTaskQueue, &result, TrackInfo::kAudioTrack});
+
+  if (mDecoder) {
+    return NS_OK;
+  }
+
+  MOZ_RELEASE_ASSERT(NS_FAILED(result), "PDM returned an invalid error code");
+
+  return result;
+}
+
+void MediaDecodeTask::InitDecoder() {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  mDecoder->Init()->Then(PDecoderTaskQueue(), __func__, this,
+                         &MediaDecodeTask::OnInitDecoderCompleted,
+                         &MediaDecodeTask::OnInitDecoderFailed);
 }
 
-void MediaDecodeTask::SampleDecoded(RefPtr<AudioData> aData) {
-  MOZ_ASSERT(!NS_IsMainThread());
-  mAudioQueue.Push(aData);
-  if (!mFirstFrameDecoded) {
-    mDecoderReader->ReadUpdatedMetadata(&mMediaInfo);
-    mFirstFrameDecoded = true;
-  }
-  RequestSample();
+void MediaDecodeTask::OnInitDecoderCompleted() {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  DoDemux();
+}
+
+void MediaDecodeTask::OnInitDecoderFailed() {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  ShutdownDecoder();
+  ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
 }
 
-void MediaDecodeTask::SampleNotDecoded(const MediaResult& aError) {
-  MOZ_ASSERT(!NS_IsMainThread());
-  if (aError == NS_ERROR_DOM_MEDIA_END_OF_STREAM) {
-    FinishDecode();
+void MediaDecodeTask::DoDemux() {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  mTrackDemuxer->GetSamples(1)->Then(PDecoderTaskQueue(), __func__, this,
+                                     &MediaDecodeTask::OnAudioDemuxCompleted,
+                                     &MediaDecodeTask::OnAudioDemuxFailed);
+}
+
+void MediaDecodeTask::OnAudioDemuxCompleted(
+    RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  mRawSamples.AppendElements(aSamples->GetSamples());
+
+  DoDemux();
+}
+
+void MediaDecodeTask::OnAudioDemuxFailed(const MediaResult& aError) {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  if (aError.Code() == NS_ERROR_DOM_MEDIA_END_OF_STREAM) {
+    DoDecode();
   } else {
-    mDecoderReader->Shutdown();
+    ShutdownDecoder();
     ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
   }
 }
 
+void MediaDecodeTask::DoDecode() {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  if (mRawSamples.IsEmpty()) {
+    DoDrain();
+    return;
+  }
+
+  RefPtr<MediaRawData> sample = std::move(mRawSamples[0]);
+
+  mDecoder->Decode(sample)->Then(PDecoderTaskQueue(), __func__, this,
+                                 &MediaDecodeTask::OnAudioDecodeCompleted,
+                                 &MediaDecodeTask::OnAudioDecodeFailed);
+
+  mRawSamples.RemoveElementAt(0);
+}
+
+void MediaDecodeTask::OnAudioDecodeCompleted(
+    MediaDataDecoder::DecodedData&& aResults) {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  for (auto&& sample : aResults) {
+    MOZ_ASSERT(sample->mType == MediaData::Type::AUDIO_DATA);
+    RefPtr<AudioData> audioData = sample->As<AudioData>();
+
+    mMediaInfo.mAudio.mRate = audioData->mRate;
+    mMediaInfo.mAudio.mChannels = audioData->mChannels;
+
+    mAudioQueue.Push(audioData.forget());
+  }
+
+  DoDecode();
+}
+
+void MediaDecodeTask::OnAudioDecodeFailed(const MediaResult& aError) {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  ShutdownDecoder();
+  ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
+}
+
+void MediaDecodeTask::DoDrain() {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  mDecoder->Drain()->Then(PDecoderTaskQueue(), __func__, this,
+                          &MediaDecodeTask::OnAudioDrainCompleted,
+                          &MediaDecodeTask::OnAudioDrainFailed);
+}
+
+void MediaDecodeTask::OnAudioDrainCompleted(
+    MediaDataDecoder::DecodedData&& aResults) {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  if (aResults.IsEmpty()) {
+    FinishDecode();
+    return;
+  }
+
+  for (auto&& sample : aResults) {
+    MOZ_ASSERT(sample->mType == MediaData::Type::AUDIO_DATA);
+    RefPtr<AudioData> audioData = sample->As<AudioData>();
+
+    mAudioQueue.Push(audioData.forget());
+  }
+  DoDrain();
+}
+
+void MediaDecodeTask::OnAudioDrainFailed(const MediaResult& aError) {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  ShutdownDecoder();
+  ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
+}
+
+void MediaDecodeTask::ShutdownDecoder() {
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  if (!mDecoder) {
+    return;
+  }
+
+  RefPtr<MediaDecodeTask> self = this;
+  mDecoder->Shutdown();
+  mDecoder = nullptr;
+}
+
 void MediaDecodeTask::FinishDecode() {
-  mDecoderReader->Shutdown();
+  MOZ_ASSERT(OnPDecoderTaskQueue());
+
+  ShutdownDecoder();
 
   uint32_t frameCount = mAudioQueue.AudioFramesCount();
   uint32_t channelCount = mMediaInfo.mAudio.mChannels;
   uint32_t sampleRate = mMediaInfo.mAudio.mRate;
 
   if (!frameCount || !channelCount || !sampleRate) {
     ReportFailureOnMainThread(WebAudioDecodeJob::InvalidContent);
     return;
@@ -447,27 +599,27 @@ void AsyncDecodeWebAudio(const char* aCo
                              WebAudioDecodeJob::UnknownContent);
     JS_free(nullptr, aBuffer);
     aDecodeJob.mContext->Dispatch(event.forget());
     return;
   }
 
   RefPtr<MediaDecodeTask> task =
       new MediaDecodeTask(*containerType, aBuffer, aLength, aDecodeJob);
-  if (!task->CreateReader()) {
+  if (!task->Init()) {
     nsCOMPtr<nsIRunnable> event =
         new ReportResultTask(aDecodeJob, &WebAudioDecodeJob::OnFailure,
                              WebAudioDecodeJob::UnknownError);
     aDecodeJob.mContext->Dispatch(event.forget());
   } else {
     // If we did this without a temporary:
-    //   task->Reader()->OwnerThread()->Dispatch(task.forget())
-    // we might evaluate the task.forget() before calling Reader(). Enforce
-    // a non-crashy order-of-operations.
-    TaskQueue* taskQueue = task->Reader()->OwnerThread();
+    //   task->PDecoderTaskQueue()->Dispatch(task.forget())
+    // we might evaluate the task.forget() before calling PDecoderTaskQueue().
+    // Enforce a non-crashy order-of-operations.
+    TaskQueue* taskQueue = task->PDecoderTaskQueue();
     nsresult rv = taskQueue->Dispatch(task.forget());
     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
     Unused << rv;
   }
 }
 
 WebAudioDecodeJob::WebAudioDecodeJob(AudioContext* aContext, Promise* aPromise,
                                      DecodeSuccessCallback* aSuccessCallback,