Bug 1315850 - Implement video decoding through CDM. r=jya draft
authorChris Pearce <cpearce@mozilla.com>
Thu, 09 Mar 2017 11:32:15 +1300
changeset 504173 b7f162515a1a32b2c344c11d0fa5c7004cec2e15
parent 504172 b6d499cafef2d6a6558b0db703b60320dea67803
child 504174 8ee1ae28a36779484717c6b105ef7730dd1896b3
push id50748
push userbmo:cpearce@mozilla.com
push dateFri, 24 Mar 2017 01:10:17 +0000
reviewersjya
bugs1315850
milestone55.0a1
Bug 1315850 - Implement video decoding through CDM. r=jya At this stage, I store video frames in memory in nsTArrays rather than in shmems just so we can get this working. Once this is working, I'll follow up with patches to switch to storing all large buffer traffic between the CDM and other processes in shmems. I'm not planning on preffing this new CDM path on until that's in place. MozReview-Commit-ID: LSTb42msWQS
dom/media/gmp/ChromiumCDMChild.cpp
dom/media/gmp/ChromiumCDMChild.h
dom/media/gmp/ChromiumCDMParent.cpp
dom/media/gmp/ChromiumCDMParent.h
dom/media/gmp/widevine-adapter/moz.build
dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp
dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h
--- a/dom/media/gmp/ChromiumCDMChild.cpp
+++ b/dom/media/gmp/ChromiumCDMChild.cpp
@@ -1,16 +1,17 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "ChromiumCDMChild.h"
 #include "GMPContentChild.h"
 #include "WidevineUtils.h"
+#include "WidevineVideoFrame.h"
 #include "GMPLog.h"
 #include "GMPPlatform.h"
 #include "mozilla/Unused.h"
 #include "nsPrintfCString.h"
 #include "base/time.h"
 
 namespace mozilla {
 namespace gmp {
@@ -462,16 +463,64 @@ ChromiumCDMChild::RecvResetVideoDecoder(
 }
 
 mozilla::ipc::IPCResult
 ChromiumCDMChild::RecvDecryptAndDecodeFrame(const CDMInputBuffer& aBuffer)
 {
   MOZ_ASSERT(IsOnMessageLoopThread());
   GMP_LOG("ChromiumCDMChild::RecvDecryptAndDecodeFrame()");
   MOZ_ASSERT(mDecoderInitialized);
+
+  // The output frame may not have the same timestamp as the frame we put in.
+  // We may need to input a number of frames before we receive output. The
+  // CDM's decoder reorders to ensure frames output are in presentation order.
+  // So we need to store the durations of the frames input, and retrieve them
+  // on output.
+  mFrameDurations.Insert(aBuffer.mTimestamp(), aBuffer.mDuration());
+
+  cdm::InputBuffer input;
+  nsTArray<cdm::SubsampleEntry> subsamples;
+  InitInputBuffer(aBuffer, subsamples, input);
+
+  WidevineVideoFrame frame;
+  cdm::Status rv = mCDM->DecryptAndDecodeFrame(input, &frame);
+  GMP_LOG("WidevineVideoDecoder::Decode(timestamp=%" PRId64 ") rv=%d",
+          input.timestamp,
+          rv);
+
+  if (rv == cdm::kSuccess) {
+    // TODO: WidevineBuffers should hold a shmem instead of a array, and we can
+    // send the handle instead of copying the array here.
+
+    gmp::CDMVideoFrame output;
+    output.mFormat() = static_cast<cdm::VideoFormat>(frame.Format());
+    output.mImageWidth() = frame.Size().width;
+    output.mImageHeight() = frame.Size().height;
+    output.mData() = Move(
+      reinterpret_cast<WidevineBuffer*>(frame.FrameBuffer())->ExtractBuffer());
+    output.mYPlane() = { frame.PlaneOffset(cdm::VideoFrame::kYPlane),
+                         frame.Stride(cdm::VideoFrame::kYPlane) };
+    output.mUPlane() = { frame.PlaneOffset(cdm::VideoFrame::kUPlane),
+                         frame.Stride(cdm::VideoFrame::kUPlane) };
+    output.mVPlane() = { frame.PlaneOffset(cdm::VideoFrame::kVPlane),
+                         frame.Stride(cdm::VideoFrame::kVPlane) };
+    output.mTimestamp() = frame.Timestamp();
+
+    uint64_t duration = 0;
+    if (mFrameDurations.Find(frame.Timestamp(), duration)) {
+      output.mDuration() = duration;
+    }
+
+    Unused << SendDecoded(output);
+  } else if (rv == cdm::kNeedMoreData) {
+    Unused << SendDecoded(gmp::CDMVideoFrame());
+  } else {
+    Unused << SendDecodeFailed(rv);
+  }
+
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
 ChromiumCDMChild::RecvDestroy()
 {
   MOZ_ASSERT(IsOnMessageLoopThread());
   GMP_LOG("ChromiumCDMChild::RecvDestroy()");
--- a/dom/media/gmp/ChromiumCDMChild.h
+++ b/dom/media/gmp/ChromiumCDMChild.h
@@ -3,16 +3,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef ChromiumCDMChild_h_
 #define ChromiumCDMChild_h_
 
 #include "mozilla/gmp/PChromiumCDMChild.h"
 #include "content_decryption_module.h"
+#include "SimpleMap.h"
 
 namespace mozilla {
 namespace gmp {
 
 class GMPContentChild;
 
 class ChromiumCDMChild : public PChromiumCDMChild
                        , public cdm::Host_8
@@ -103,15 +104,18 @@ protected:
     const CDMInputBuffer& aBuffer) override;
   ipc::IPCResult RecvDestroy() override;
 
   void DecryptFailed(uint32_t aId, cdm::Status aStatus);
 
   GMPContentChild* mPlugin = nullptr;
   cdm::ContentDecryptionModule_8* mCDM = nullptr;
 
+  typedef SimpleMap<uint64_t> DurationMap;
+  DurationMap mFrameDurations;
+
   bool mDecoderInitialized = false;
 };
 
 } // namespace gmp
 } // namespace mozilla
 
 #endif // ChromiumCDMChild_h_
--- a/dom/media/gmp/ChromiumCDMParent.cpp
+++ b/dom/media/gmp/ChromiumCDMParent.cpp
@@ -440,22 +440,77 @@ ChromiumCDMParent::RecvDecrypted(const u
     }
   }
   return IPC_OK();
 }
 
 ipc::IPCResult
 ChromiumCDMParent::RecvDecoded(const CDMVideoFrame& aFrame)
 {
+  VideoData::YCbCrBuffer b;
+  nsTArray<uint8_t> data;
+  data = aFrame.mData();
+
+  if (data.IsEmpty()) {
+    mDecodePromise.ResolveIfExists(nsTArray<RefPtr<MediaData>>(), __func__);
+    return IPC_OK();
+  }
+
+  b.mPlanes[0].mData = data.Elements();
+  b.mPlanes[0].mWidth = aFrame.mImageWidth();
+  b.mPlanes[0].mHeight = aFrame.mImageHeight();
+  b.mPlanes[0].mStride = aFrame.mYPlane().mStride();
+  b.mPlanes[0].mOffset = aFrame.mYPlane().mPlaneOffset();
+  b.mPlanes[0].mSkip = 0;
+
+  b.mPlanes[1].mData = data.Elements();
+  b.mPlanes[1].mWidth = (aFrame.mImageWidth() + 1) / 2;
+  b.mPlanes[1].mHeight = (aFrame.mImageHeight() + 1) / 2;
+  b.mPlanes[1].mStride = aFrame.mUPlane().mStride();
+  b.mPlanes[1].mOffset = aFrame.mUPlane().mPlaneOffset();
+  b.mPlanes[1].mSkip = 0;
+
+  b.mPlanes[2].mData = data.Elements();
+  b.mPlanes[2].mWidth = (aFrame.mImageWidth() + 1) / 2;
+  b.mPlanes[2].mHeight = (aFrame.mImageHeight() + 1) / 2;
+  b.mPlanes[2].mStride = aFrame.mVPlane().mStride();
+  b.mPlanes[2].mOffset = aFrame.mVPlane().mPlaneOffset();
+  b.mPlanes[2].mSkip = 0;
+
+  gfx::IntRect pictureRegion(0, 0, aFrame.mImageWidth(), aFrame.mImageHeight());
+  RefPtr<VideoData> v = VideoData::CreateAndCopyData(mVideoInfo,
+                                                     mImageContainer,
+                                                     mLastStreamOffset,
+                                                     aFrame.mTimestamp(),
+                                                     aFrame.mDuration(),
+                                                     b,
+                                                     false,
+                                                     -1,
+                                                     pictureRegion);
+
+  RefPtr<ChromiumCDMParent> self = this;
+  if (v) {
+    mDecodePromise.ResolveIfExists({ Move(v) }, __func__);
+  } else {
+    mDecodePromise.RejectIfExists(
+      MediaResult(NS_ERROR_OUT_OF_MEMORY,
+                  RESULT_DETAIL("CallBack::CreateAndCopyData")),
+      __func__);
+  }
+
   return IPC_OK();
 }
 
 ipc::IPCResult
 ChromiumCDMParent::RecvDecodeFailed(const uint32_t& aStatus)
 {
+  mDecodePromise.RejectIfExists(
+    MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+                RESULT_DETAIL("ChromiumCDMParent::RecvDecodeFailed")),
+    __func__);
   return IPC_OK();
 }
 
 ipc::IPCResult
 ChromiumCDMParent::RecvShutdown()
 {
   GMP_LOG("ChromiumCDMParent::RecvShutdown(this=%p)", this);
   // TODO: SendDestroy(), call Terminated.
@@ -465,25 +520,30 @@ ChromiumCDMParent::RecvShutdown()
 void
 ChromiumCDMParent::ActorDestroy(ActorDestroyReason aWhy)
 {
   GMP_LOG("ChromiumCDMParent::ActorDestroy(this=%p, reason=%d)", this, aWhy);
 }
 
 RefPtr<MediaDataDecoder::InitPromise>
 ChromiumCDMParent::InitializeVideoDecoder(
-  const gmp::CDMVideoDecoderConfig& aConfig)
+  const gmp::CDMVideoDecoderConfig& aConfig,
+  const VideoInfo& aInfo,
+  RefPtr<layers::ImageContainer> aImageContainer)
 {
   if (!SendInitializeVideoDecoder(aConfig)) {
     return MediaDataDecoder::InitPromise::CreateAndReject(
       MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
                   RESULT_DETAIL("Failed to send init video decoder to CDM")),
       __func__);
   }
 
+  mImageContainer = aImageContainer;
+  mVideoInfo = aInfo;
+
   return mInitVideoDecoderPromise.Ensure(__func__);
 }
 
 ipc::IPCResult
 ChromiumCDMParent::RecvOnDecoderInitDone(const uint32_t& aStatus)
 {
   GMP_LOG("ChromiumCDMParent::RecvOnDecoderInitDone(this=%p, status=%u)",
           this,
@@ -495,10 +555,36 @@ ChromiumCDMParent::RecvOnDecoderInitDone
       MediaResult(
         NS_ERROR_DOM_MEDIA_FATAL_ERR,
         RESULT_DETAIL("CDM init decode failed with %" PRIu32, aStatus)),
       __func__);
   }
   return IPC_OK();
 }
 
+RefPtr<MediaDataDecoder::DecodePromise>
+ChromiumCDMParent::DecryptAndDecodeFrame(MediaRawData* aSample)
+{
+  CDMInputBuffer buffer;
+
+  if (!InitCDMInputBuffer(buffer, aSample)) {
+    return MediaDataDecoder::DecodePromise::CreateAndReject(
+      MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Failed to init CDM buffer."),
+      __func__);
+  }
+
+  mLastStreamOffset = aSample->mOffset;
+
+  if (!SendDecryptAndDecodeFrame(buffer)) {
+    GMP_LOG(
+      "ChromiumCDMParent::Decrypt(this=%p) failed to send decrypt message.",
+      this);
+    return MediaDataDecoder::DecodePromise::CreateAndReject(
+      MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+                  "Failed to send decrypt to CDM process."),
+      __func__);
+  }
+
+  return mDecodePromise.Ensure(__func__);
+}
+
 } // namespace gmp
 } // namespace mozilla
--- a/dom/media/gmp/ChromiumCDMParent.h
+++ b/dom/media/gmp/ChromiumCDMParent.h
@@ -9,16 +9,17 @@
 #include "DecryptJob.h"
 #include "GMPCrashHelper.h"
 #include "GMPCrashHelperHolder.h"
 #include "GMPMessageUtils.h"
 #include "mozilla/gmp/PChromiumCDMParent.h"
 #include "mozilla/RefPtr.h"
 #include "nsDataHashtable.h"
 #include "PlatformDecoderModule.h"
+#include "ImageContainer.h"
 
 namespace mozilla {
 
 class MediaRawData;
 class ChromiumCDMProxy;
 
 namespace gmp {
 
@@ -56,17 +57,22 @@ public:
 
   void RemoveSession(const nsCString& aSessionId, uint32_t aPromiseId);
 
   RefPtr<DecryptPromise> Decrypt(MediaRawData* aSample);
 
   // TODO: Add functions for clients to send data to CDM, and
   // a Close() function.
   RefPtr<MediaDataDecoder::InitPromise> InitializeVideoDecoder(
-    const gmp::CDMVideoDecoderConfig& aConfig);
+    const gmp::CDMVideoDecoderConfig& aConfig,
+    const VideoInfo& aInfo,
+    RefPtr<layers::ImageContainer> aImageContainer);
+
+  RefPtr<MediaDataDecoder::DecodePromise> DecryptAndDecodeFrame(
+    MediaRawData* aSample);
 
 protected:
   ~ChromiumCDMParent() {}
 
   ipc::IPCResult Recv__delete__() override;
   ipc::IPCResult RecvOnResolveNewSessionPromise(
     const uint32_t& aPromiseId,
     const nsCString& aSessionId) override;
@@ -109,14 +115,19 @@ protected:
   // Note: this pointer is a weak reference because otherwise it would cause
   // a cycle, as ChromiumCDMProxy has a strong reference to the
   // ChromiumCDMParent.
   ChromiumCDMProxy* mProxy = nullptr;
   nsDataHashtable<nsUint32HashKey, uint32_t> mPromiseToCreateSessionToken;
   nsTArray<RefPtr<DecryptJob>> mDecrypts;
 
   MozPromiseHolder<MediaDataDecoder::InitPromise> mInitVideoDecoderPromise;
+  MozPromiseHolder<MediaDataDecoder::DecodePromise> mDecodePromise;
+
+  RefPtr<layers::ImageContainer> mImageContainer;
+  VideoInfo mVideoInfo;
+  uint64_t mLastStreamOffset = 0;
 };
 
 } // namespace gmp
 } // namespace mozilla
 
 #endif // ChromiumCDMParent_h_
--- a/dom/media/gmp/widevine-adapter/moz.build
+++ b/dom/media/gmp/widevine-adapter/moz.build
@@ -11,17 +11,18 @@ SOURCES += [
     'WidevineFileIO.cpp',
     'WidevineUtils.cpp',
     'WidevineVideoDecoder.cpp',
     'WidevineVideoFrame.cpp',
 ]
 
 EXPORTS += [
     'WidevineDecryptor.h',
-    'WidevineUtils.h'
+    'WidevineUtils.h',
+    'WidevineVideoFrame.h'
 ]
 
 FINAL_LIBRARY = 'xul'
 
 LOCAL_INCLUDES += [
     '/dom/media/gmp',
 ]
 
--- a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp
+++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp
@@ -63,41 +63,60 @@ ChromiumCDMVideoDecoder::Init()
   }
 
   gmp::CDMVideoDecoderConfig config;
   if (MP4Decoder::IsH264(mConfig.mMimeType)) {
     config.mCodec() = cdm::VideoDecoderConfig::kCodecH264;
     config.mProfile() =
       ToCDMH264Profile(mConfig.mExtraData->SafeElementAt(1, 0));
     config.mExtraData() = *mConfig.mExtraData;
+    mConvertToAnnexB = true;
   } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
     config.mCodec() = cdm::VideoDecoderConfig::kCodecVp8;
     config.mProfile() = cdm::VideoDecoderConfig::kProfileNotNeeded;
   } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
     config.mCodec() = cdm::VideoDecoderConfig::kCodecVp9;
     config.mProfile() = cdm::VideoDecoderConfig::kProfileNotNeeded;
   } else {
     return MediaDataDecoder::InitPromise::CreateAndReject(
       NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
   }
   config.mImageWidth() = mConfig.mImage.width;
   config.mImageHeight() = mConfig.mImage.height;
 
   RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
-  return InvokeAsync(mGMPThread, __func__, [cdm, config]() {
-    return cdm->InitializeVideoDecoder(config);
-  });
+  VideoInfo info = mConfig;
+  RefPtr<layers::ImageContainer> imageContainer = mImageContainer;
+  return InvokeAsync(
+    mGMPThread, __func__, [cdm, config, info, imageContainer]() {
+      return cdm->InitializeVideoDecoder(config, info, imageContainer);
+    });
+}
+
+const char*
+ChromiumCDMVideoDecoder::GetDescriptionName() const
+{
+  return "Chromium CDM video decoder";
+}
+
+MediaDataDecoder::ConversionRequired
+ChromiumCDMVideoDecoder::NeedsConversion() const
+{
+  return mConvertToAnnexB ? ConversionRequired::kNeedAnnexB
+                          : ConversionRequired::kNeedNone;
 }
 
 RefPtr<MediaDataDecoder::DecodePromise>
 ChromiumCDMVideoDecoder::Decode(MediaRawData* aSample)
 {
-  return DecodePromise::CreateAndReject(
-    MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("Unimplemented")),
-    __func__);
+  RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
+  RefPtr<MediaRawData> sample = aSample;
+  return InvokeAsync(mGMPThread, __func__, [cdm, sample]() {
+    return cdm->DecryptAndDecodeFrame(sample);
+  });
 }
 
 RefPtr<MediaDataDecoder::FlushPromise>
 ChromiumCDMVideoDecoder::Flush()
 {
   return FlushPromise::CreateAndReject(
     MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, RESULT_DETAIL("Unimplemented")),
     __func__);
--- a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h
+++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h
@@ -21,27 +21,26 @@ public:
   ChromiumCDMVideoDecoder(const GMPVideoDecoderParams& aParams,
                           CDMProxy* aCDMProxy);
 
   RefPtr<InitPromise> Init() override;
   RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
   RefPtr<FlushPromise> Flush() override;
   RefPtr<DecodePromise> Drain() override;
   RefPtr<ShutdownPromise> Shutdown() override;
-  const char* GetDescriptionName() const override
-  {
-    return "Chromium CDM video decoder";
-  }
+  const char* GetDescriptionName() const override;
+  ConversionRequired NeedsConversion() const override;
 
 private:
   ~ChromiumCDMVideoDecoder();
 
   RefPtr<gmp::ChromiumCDMParent> mCDMParent;
   const VideoInfo mConfig;
   RefPtr<GMPCrashHelper> mCrashHelper;
   RefPtr<AbstractThread> mGMPThread;
   RefPtr<layers::ImageContainer> mImageContainer;
   MozPromiseHolder<InitPromise> mInitPromise;
+  bool mConvertToAnnexB = false;
 };
 
 } // mozilla
 
 #endif // ChromiumCDMVideoDecoder_h_