Bug 1061525 - Part 7: Add software backed NV12 images support. r=mattwoodrow
authorJean-Yves Avenard <jyavenard@mozilla.com>
Thu, 06 Aug 2015 17:26:42 +1000
changeset 289664 f57e11edbde2c0a25953b141dada2aa4d07dc250
parent 289663 2eb3603f5c9f43570fc5bd1cc27a53f24fecf85c
child 289665 5dffd92d792de2ab1c2fef316b3eae2dccf253ef
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmattwoodrow
bugs1061525
milestone42.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1061525 - Part 7: Add software backed NV12 images support. r=mattwoodrow Currently unused.
dom/media/platforms/apple/AppleVDADecoder.cpp
dom/media/platforms/apple/AppleVDADecoder.h
dom/media/platforms/apple/AppleVTDecoder.cpp
--- a/dom/media/platforms/apple/AppleVDADecoder.cpp
+++ b/dom/media/platforms/apple/AppleVDADecoder.cpp
@@ -35,18 +35,19 @@ AppleVDADecoder::AppleVDADecoder(const V
                                layers::ImageContainer* aImageContainer)
   : mTaskQueue(aVideoTaskQueue)
   , mCallback(aCallback)
   , mImageContainer(aImageContainer)
   , mPictureWidth(aConfig.mImage.width)
   , mPictureHeight(aConfig.mImage.height)
   , mDisplayWidth(aConfig.mDisplay.width)
   , mDisplayHeight(aConfig.mDisplay.height)
+  , mUseSoftwareImages(false)
+  , mIs106(!nsCocoaFeatures::OnLionOrLater())
   , mDecoder(nullptr)
-  , mIs106(!nsCocoaFeatures::OnLionOrLater())
 {
   MOZ_COUNT_CTOR(AppleVDADecoder);
   // TODO: Verify aConfig.mime_type.
 
   mExtraData = aConfig.mExtraData;
   mMaxRefFrames = 4;
   // Retrieve video dimensions from H264 SPS NAL.
   mp4_demuxer::SPSData spsdata;
@@ -243,52 +244,111 @@ AppleVDADecoder::ClearReorderedFrames()
   }
 }
 
 // Copy and return a decoded frame.
 nsresult
 AppleVDADecoder::OutputFrame(CVPixelBufferRef aImage,
                              nsAutoPtr<AppleVDADecoder::AppleFrameRef> aFrameRef)
 {
-  IOSurfacePtr surface = MacIOSurfaceLib::CVPixelBufferGetIOSurface(aImage);
-  MOZ_ASSERT(surface, "Decoder didn't return an IOSurface backed buffer");
-
   LOG("mp4 output frame %lld dts %lld pts %lld duration %lld us%s",
-    aFrameRef->byte_offset,
-    aFrameRef->decode_timestamp.ToMicroseconds(),
-    aFrameRef->composition_timestamp.ToMicroseconds(),
-    aFrameRef->duration.ToMicroseconds(),
-    aFrameRef->is_sync_point ? " keyframe" : ""
+      aFrameRef->byte_offset,
+      aFrameRef->decode_timestamp.ToMicroseconds(),
+      aFrameRef->composition_timestamp.ToMicroseconds(),
+      aFrameRef->duration.ToMicroseconds(),
+      aFrameRef->is_sync_point ? " keyframe" : ""
   );
 
-  nsRefPtr<MacIOSurface> macSurface = new MacIOSurface(surface);
+  // Where our resulting image will end up.
+  nsRefPtr<VideoData> data;
   // Bounds.
   VideoInfo info;
   info.mDisplay = nsIntSize(mDisplayWidth, mDisplayHeight);
   gfx::IntRect visible = gfx::IntRect(0,
                                       0,
                                       mPictureWidth,
                                       mPictureHeight);
 
-  nsRefPtr<layers::Image> image =
-    mImageContainer->CreateImage(ImageFormat::MAC_IOSURFACE);
-  layers::MacIOSurfaceImage* videoImage =
-    static_cast<layers::MacIOSurfaceImage*>(image.get());
-  videoImage->SetSurface(macSurface);
+  if (mUseSoftwareImages) {
+    size_t width = CVPixelBufferGetWidth(aImage);
+    size_t height = CVPixelBufferGetHeight(aImage);
+    DebugOnly<size_t> planes = CVPixelBufferGetPlaneCount(aImage);
+    MOZ_ASSERT(planes == 2, "Likely not NV12 format and it must be.");
+
+    VideoData::YCbCrBuffer buffer;
 
-  nsRefPtr<VideoData> data;
-  data = VideoData::CreateFromImage(info,
-                                    mImageContainer,
-                                    aFrameRef->byte_offset,
-                                    aFrameRef->composition_timestamp.ToMicroseconds(),
-                                    aFrameRef->duration.ToMicroseconds(),
-                                    image.forget(),
-                                    aFrameRef->is_sync_point,
-                                    aFrameRef->decode_timestamp.ToMicroseconds(),
-                                    visible);
+    // Lock the returned image data.
+    CVReturn rv = CVPixelBufferLockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly);
+    if (rv != kCVReturnSuccess) {
+      NS_ERROR("error locking pixel data");
+      mCallback->Error();
+      return NS_ERROR_FAILURE;
+    }
+    // Y plane.
+    buffer.mPlanes[0].mData =
+      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 0));
+    buffer.mPlanes[0].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 0);
+    buffer.mPlanes[0].mWidth = width;
+    buffer.mPlanes[0].mHeight = height;
+    buffer.mPlanes[0].mOffset = 0;
+    buffer.mPlanes[0].mSkip = 0;
+    // Cb plane.
+    buffer.mPlanes[1].mData =
+      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 1));
+    buffer.mPlanes[1].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 1);
+    buffer.mPlanes[1].mWidth = (width+1) / 2;
+    buffer.mPlanes[1].mHeight = (height+1) / 2;
+    buffer.mPlanes[1].mOffset = 0;
+    buffer.mPlanes[1].mSkip = 1;
+    // Cr plane.
+    buffer.mPlanes[2].mData =
+      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 1));
+    buffer.mPlanes[2].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 1);
+    buffer.mPlanes[2].mWidth = (width+1) / 2;
+    buffer.mPlanes[2].mHeight = (height+1) / 2;
+    buffer.mPlanes[2].mOffset = 1;
+    buffer.mPlanes[2].mSkip = 1;
+
+    // Copy the image data into our own format.
+    data =
+      VideoData::Create(info,
+                        mImageContainer,
+                        nullptr,
+                        aFrameRef->byte_offset,
+                        aFrameRef->composition_timestamp.ToMicroseconds(),
+                        aFrameRef->duration.ToMicroseconds(),
+                        buffer,
+                        aFrameRef->is_sync_point,
+                        aFrameRef->decode_timestamp.ToMicroseconds(),
+                        visible);
+    // Unlock the returned image data.
+    CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly);
+  } else {
+    IOSurfacePtr surface = MacIOSurfaceLib::CVPixelBufferGetIOSurface(aImage);
+    MOZ_ASSERT(surface, "Decoder didn't return an IOSurface backed buffer");
+
+    nsRefPtr<MacIOSurface> macSurface = new MacIOSurface(surface);
+
+    nsRefPtr<layers::Image> image =
+      mImageContainer->CreateImage(ImageFormat::MAC_IOSURFACE);
+    layers::MacIOSurfaceImage* videoImage =
+      static_cast<layers::MacIOSurfaceImage*>(image.get());
+    videoImage->SetSurface(macSurface);
+
+    data =
+      VideoData::CreateFromImage(info,
+                                 mImageContainer,
+                                 aFrameRef->byte_offset,
+                                 aFrameRef->composition_timestamp.ToMicroseconds(),
+                                 aFrameRef->duration.ToMicroseconds(),
+                                 image.forget(),
+                                 aFrameRef->is_sync_point,
+                                 aFrameRef->decode_timestamp.ToMicroseconds(),
+                                 visible);
+  }
 
   if (!data) {
     NS_ERROR("Couldn't create VideoData for frame");
     mCallback->Error();
     return NS_ERROR_FAILURE;
   }
 
   // Frames come out in DTS order but we need to output them
@@ -454,37 +514,52 @@ AppleVDADecoder::CreateDecoderSpecificat
                             ArrayLength(decoderKeys),
                             &kCFTypeDictionaryKeyCallBacks,
                             &kCFTypeDictionaryValueCallBacks);
 }
 
 CFDictionaryRef
 AppleVDADecoder::CreateOutputConfiguration()
 {
+  // Output format type:
+  SInt32 PixelFormatTypeValue = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
+  AutoCFRelease<CFNumberRef> PixelFormatTypeNumber =
+    CFNumberCreate(kCFAllocatorDefault,
+                   kCFNumberSInt32Type,
+                   &PixelFormatTypeValue);
+
+  if (mUseSoftwareImages) {
+    const void* outputKeys[] = { kCVPixelBufferPixelFormatTypeKey };
+    const void* outputValues[] = { PixelFormatTypeNumber };
+    static_assert(ArrayLength(outputKeys) == ArrayLength(outputValues),
+                  "Non matching keys/values array size");
+
+    return CFDictionaryCreate(kCFAllocatorDefault,
+                              outputKeys,
+                              outputValues,
+                              ArrayLength(outputKeys),
+                              &kCFTypeDictionaryKeyCallBacks,
+                              &kCFTypeDictionaryValueCallBacks);
+  }
+
   // Construct IOSurface Properties
   const void* IOSurfaceKeys[] = { MacIOSurfaceLib::kPropIsGlobal };
   const void* IOSurfaceValues[] = { kCFBooleanTrue };
   static_assert(ArrayLength(IOSurfaceKeys) == ArrayLength(IOSurfaceValues),
                 "Non matching keys/values array size");
 
   // Contruct output configuration.
   AutoCFRelease<CFDictionaryRef> IOSurfaceProperties =
     CFDictionaryCreate(kCFAllocatorDefault,
                        IOSurfaceKeys,
                        IOSurfaceValues,
                        ArrayLength(IOSurfaceKeys),
                        &kCFTypeDictionaryKeyCallBacks,
                        &kCFTypeDictionaryValueCallBacks);
 
-  SInt32 PixelFormatTypeValue = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
-  AutoCFRelease<CFNumberRef> PixelFormatTypeNumber =
-    CFNumberCreate(kCFAllocatorDefault,
-                   kCFNumberSInt32Type,
-                   &PixelFormatTypeValue);
-
   const void* outputKeys[] = { kCVPixelBufferIOSurfacePropertiesKey,
                                kCVPixelBufferPixelFormatTypeKey,
                                kCVPixelBufferOpenGLCompatibilityKey };
   const void* outputValues[] = { IOSurfaceProperties,
                                  PixelFormatTypeNumber,
                                  kCFBooleanTrue };
   static_assert(ArrayLength(outputKeys) == ArrayLength(outputValues),
                 "Non matching keys/values array size");
--- a/dom/media/platforms/apple/AppleVDADecoder.h
+++ b/dom/media/platforms/apple/AppleVDADecoder.h
@@ -94,20 +94,21 @@ public:
   MediaDataDecoderCallback* mCallback;
   nsRefPtr<layers::ImageContainer> mImageContainer;
   ReorderQueue mReorderQueue;
   uint32_t mPictureWidth;
   uint32_t mPictureHeight;
   uint32_t mDisplayWidth;
   uint32_t mDisplayHeight;
   uint32_t mMaxRefFrames;
+  bool mUseSoftwareImages;
+  bool mIs106;
 
 private:
   VDADecoder mDecoder;
-  bool mIs106;
 
   // Method to pass a frame to VideoToolbox for decoding.
   nsresult SubmitFrame(MediaRawData* aSample);
   // Method to set up the decompression session.
   nsresult InitializeSession();
   CFDictionaryRef CreateDecoderSpecification();
 };
 
--- a/dom/media/platforms/apple/AppleVTDecoder.cpp
+++ b/dom/media/platforms/apple/AppleVTDecoder.cpp
@@ -7,17 +7,16 @@
 #include <CoreFoundation/CFString.h>
 
 #include "AppleCMLinker.h"
 #include "AppleUtils.h"
 #include "AppleVTDecoder.h"
 #include "AppleVTLinker.h"
 #include "mp4_demuxer/H264.h"
 #include "MediaData.h"
-#include "MacIOSurfaceImage.h"
 #include "mozilla/ArrayUtils.h"
 #include "nsAutoPtr.h"
 #include "nsThreadUtils.h"
 #include "mozilla/Logging.h"
 #include "VideoUtils.h"
 #include "gfxPlatform.h"
 
 PRLogModuleInfo* GetAppleMediaLog();