Bug 1540748 - part1 : Android decoder should throw an error when the decoded sample's time is smaller than the time of first demuxed sample. r=jolin
☠☠ backed out by 420e18a75314 ☠ ☠
authorAlastor Wu <alwu@mozilla.com>
Thu, 25 Apr 2019 21:47:53 +0000
changeset 530438 8c9fc50e48c2a04b4167ab45884fd3f7dbaee763
parent 530437 8754c7d0d1167eb3e47d36a616e570ed8188ae96
child 530439 57b12599afe20163b7abd2b0e9dd23f08e2268c4
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjolin
bugs1540748
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1540748 - part1 : Android decoder should throw an error when the decoded sample's time is smaller than the time of first demuxed sample. r=jolin Considering that the audio sample's time is always increased, the decoded sample's time decoder returns should always be equal or larger than its demuxed time. When the decoded sample's time is smaller than the time of first demuxed sample, that time would probably cause a problem so we should throw an error for that. Differential Revision: https://phabricator.services.mozilla.com/D28167
dom/media/platforms/android/RemoteDataDecoder.cpp
dom/media/platforms/android/RemoteDataDecoder.h
--- a/dom/media/platforms/android/RemoteDataDecoder.cpp
+++ b/dom/media/platforms/android/RemoteDataDecoder.cpp
@@ -168,24 +168,28 @@ class RemoteVideoDecoder : public Remote
       mSeekTarget.reset();
       mLatestOutputTime.reset();
       return RemoteDataDecoder::ProcessFlush();
     });
   }
 
   RefPtr<MediaDataDecoder::DecodePromise> Decode(
       MediaRawData* aSample) override {
-    const VideoInfo* config =
-        aSample->mTrackInfo ? aSample->mTrackInfo->GetAsVideoInfo() : &mConfig;
-    MOZ_ASSERT(config);
+    RefPtr<RemoteVideoDecoder> self = this;
+    RefPtr<MediaRawData> sample = aSample;
+    return InvokeAsync(mTaskQueue, __func__, [self, this, sample]() {
+      const VideoInfo* config =
+          sample->mTrackInfo ? sample->mTrackInfo->GetAsVideoInfo() : &mConfig;
+      MOZ_ASSERT(config);
 
-    InputInfo info(aSample->mDuration.ToMicroseconds(), config->mImage,
-                   config->mDisplay);
-    mInputInfos.Insert(aSample->mTime.ToMicroseconds(), info);
-    return RemoteDataDecoder::Decode(aSample);
+      InputInfo info(sample->mDuration.ToMicroseconds(), config->mImage,
+                     config->mDisplay);
+      mInputInfos.Insert(sample->mTime.ToMicroseconds(), info);
+      return RemoteDataDecoder::ProcessDecode(sample);
+    });
   }
 
   bool SupportDecoderRecycling() const override {
     return mIsCodecSupportAdaptivePlayback;
   }
 
   void SetSeekThreshold(const TimeUnit& aTime) override {
     RefPtr<RemoteVideoDecoder> self = this;
@@ -359,16 +363,36 @@ class RemoteAudioDecoder : public Remote
     if (mJavaDecoder == nullptr) {
       return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
                                           __func__);
     }
 
     return InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__);
   }
 
+  RefPtr<FlushPromise> Flush() override {
+    RefPtr<RemoteAudioDecoder> self = this;
+    return InvokeAsync(mTaskQueue, __func__, [self, this]() {
+      mFirstDemuxedSampleTime.reset();
+      return RemoteDataDecoder::ProcessFlush();
+    });
+  }
+
+  RefPtr<DecodePromise> Decode(MediaRawData* aSample) override {
+    RefPtr<RemoteAudioDecoder> self = this;
+    RefPtr<MediaRawData> sample = aSample;
+    return InvokeAsync(mTaskQueue, __func__, [self, sample, this]() {
+      if (mFirstDemuxedSampleTime.isNothing()) {
+        MOZ_ASSERT(sample->mTime.IsValid());
+        mFirstDemuxedSampleTime.emplace(sample->mTime);
+      }
+      return RemoteDataDecoder::ProcessDecode(sample);
+    });
+  }
+
  private:
   class CallbacksSupport final : public JavaCallbacksSupport {
    public:
     explicit CallbacksSupport(RemoteAudioDecoder* aDecoder)
         : mDecoder(aDecoder) {}
 
     void HandleInput(int64_t aTimestamp, bool aProcessed) override {
       mDecoder->UpdateInputStatus(aTimestamp, aProcessed);
@@ -403,16 +427,20 @@ class RemoteAudioDecoder : public Remote
     void HandleError(const MediaResult& aError) override {
       mDecoder->Error(aError);
     }
 
    private:
     RemoteAudioDecoder* mDecoder;
   };
 
+  bool IsSampleTimeSmallerThanFirstDemuxedSampleTime(int64_t aTime) const {
+    return TimeUnit::FromMicroseconds(aTime) < mFirstDemuxedSampleTime.ref();
+  }
+
   // Param and LocalRef are only valid for the duration of a JNI method call.
   // Use GlobalRef as the parameter type to keep the Java object referenced
   // until running.
   void ProcessOutput(Sample::GlobalRef&& aSample,
                      SampleBuffer::GlobalRef&& aBuffer) {
     if (!mTaskQueue->IsCurrentThreadIn()) {
       nsresult rv = mTaskQueue->Dispatch(
           NewRunnableMethod<Sample::GlobalRef&&, SampleBuffer::GlobalRef&&>(
@@ -443,17 +471,18 @@ class RemoteAudioDecoder : public Remote
     ok &= NS_SUCCEEDED(info->Offset(&offset));
 
     int64_t presentationTimeUs;
     ok &= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs));
 
     int32_t size;
     ok &= NS_SUCCEEDED(info->Size(&size));
 
-    if (!ok) {
+    if (!ok ||
+        IsSampleTimeSmallerThanFirstDemuxedSampleTime(presentationTimeUs)) {
       Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__));
       return;
     }
 
     if (size > 0) {
 #ifdef MOZ_SAMPLE_TYPE_S16
       const int32_t numSamples = size / 2;
 #else
@@ -495,16 +524,17 @@ class RemoteAudioDecoder : public Remote
     AssertOnTaskQueue();
 
     mOutputChannels = aChannels;
     mOutputSampleRate = aSampleRate;
   }
 
   int32_t mOutputChannels;
   int32_t mOutputSampleRate;
+  Maybe<TimeUnit> mFirstDemuxedSampleTime;
 };
 
 already_AddRefed<MediaDataDecoder> RemoteDataDecoder::CreateAudioDecoder(
     const CreateDecoderParams& aParams, const nsString& aDrmStubId,
     CDMProxy* aProxy) {
   const AudioInfo& config = aParams.AudioConfig();
   MediaFormat::LocalRef format;
   NS_ENSURE_SUCCESS(
@@ -677,33 +707,37 @@ static CryptoInfo::LocalRef GetCryptoInf
   cryptoInfo->Set(numSubSamples, numBytesOfPlainData, numBytesOfEncryptedData,
                   keyId, iv, MediaCodec::CRYPTO_MODE_AES_CTR);
 
   return cryptoInfo;
 }
 
 RefPtr<MediaDataDecoder::DecodePromise> RemoteDataDecoder::Decode(
     MediaRawData* aSample) {
-  MOZ_ASSERT(aSample != nullptr);
-
   RefPtr<RemoteDataDecoder> self = this;
   RefPtr<MediaRawData> sample = aSample;
-  return InvokeAsync(mTaskQueue, __func__, [self, sample]() {
-    jni::ByteBuffer::LocalRef bytes = jni::ByteBuffer::New(
-        const_cast<uint8_t*>(sample->Data()), sample->Size());
+  return InvokeAsync(mTaskQueue, __func__, [self, this, sample]() {
+    return RemoteDataDecoder::ProcessDecode(sample);
+  });
+}
 
-    self->SetState(State::DRAINABLE);
-    self->mInputBufferInfo->Set(0, sample->Size(),
-                                sample->mTime.ToMicroseconds(), 0);
-    return self->mJavaDecoder->Input(bytes, self->mInputBufferInfo,
-                                     GetCryptoInfoFromSample(sample))
-               ? self->mDecodePromise.Ensure(__func__)
-               : DecodePromise::CreateAndReject(
-                     MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
-  });
+RefPtr<MediaDataDecoder::DecodePromise> RemoteDataDecoder::ProcessDecode(
+    MediaRawData* aSample) {
+  AssertOnTaskQueue();
+  MOZ_ASSERT(aSample != nullptr);
+  jni::ByteBuffer::LocalRef bytes = jni::ByteBuffer::New(
+      const_cast<uint8_t*>(aSample->Data()), aSample->Size());
+
+  SetState(State::DRAINABLE);
+  mInputBufferInfo->Set(0, aSample->Size(), aSample->mTime.ToMicroseconds(), 0);
+  return mJavaDecoder->Input(bytes, mInputBufferInfo,
+                             GetCryptoInfoFromSample(aSample))
+             ? mDecodePromise.Ensure(__func__)
+             : DecodePromise::CreateAndReject(
+                   MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
 }
 
 void RemoteDataDecoder::UpdatePendingInputStatus(PendingOp aOp) {
   AssertOnTaskQueue();
   switch (aOp) {
     case PendingOp::INCREASE:
       mNumPendingInputs++;
       break;
--- a/dom/media/platforms/android/RemoteDataDecoder.h
+++ b/dom/media/platforms/android/RemoteDataDecoder.h
@@ -38,16 +38,17 @@ class RemoteDataDecoder : public MediaDa
  protected:
   virtual ~RemoteDataDecoder() {}
   RemoteDataDecoder(MediaData::Type aType, const nsACString& aMimeType,
                     java::sdk::MediaFormat::Param aFormat,
                     const nsString& aDrmStubId, TaskQueue* aTaskQueue);
 
   // Methods only called on mTaskQueue.
   RefPtr<FlushPromise> ProcessFlush();
+  RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample);
   RefPtr<ShutdownPromise> ProcessShutdown();
   void UpdateInputStatus(int64_t aTimestamp, bool aProcessed);
   void UpdateOutputStatus(RefPtr<MediaData>&& aSample);
   void ReturnDecodedData();
   void DrainComplete();
   void Error(const MediaResult& aError);
   void AssertOnTaskQueue() { MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); }