Bug 1061525: Add software backed NV12 images support. r=mattwoodrow, a=rkothari
authorJean-Yves Avenard <jyavenard@mozilla.com>
Wed, 19 Aug 2015 16:33:24 +1000
changeset 282020 34a6fcd4793a
parent 282019 790166155c29
child 282021 4a88f07fb036
push id4986
push userjyavenard@mozilla.com
push dateMon, 24 Aug 2015 23:41:29 +0000
treeherdermozilla-beta@34a6fcd4793a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmattwoodrow, rkothari
bugs1061525
milestone41.0
Bug 1061525: Add software backed NV12 images support. r=mattwoodrow, a=rkothari Enabled by default
dom/media/platforms/apple/AppleVDADecoder.cpp
dom/media/platforms/apple/AppleVDADecoder.h
dom/media/platforms/apple/AppleVTDecoder.cpp
--- a/dom/media/platforms/apple/AppleVDADecoder.cpp
+++ b/dom/media/platforms/apple/AppleVDADecoder.cpp
@@ -35,18 +35,19 @@ AppleVDADecoder::AppleVDADecoder(const V
                                layers::ImageContainer* aImageContainer)
   : mTaskQueue(aVideoTaskQueue)
   , mCallback(aCallback)
   , mImageContainer(aImageContainer)
   , mPictureWidth(aConfig.mImage.width)
   , mPictureHeight(aConfig.mImage.height)
   , mDisplayWidth(aConfig.mDisplay.width)
   , mDisplayHeight(aConfig.mDisplay.height)
+  , mUseSoftwareImages(true)
+  , mIs106(!nsCocoaFeatures::OnLionOrLater())
   , mDecoder(nullptr)
-  , mIs106(!nsCocoaFeatures::OnLionOrLater())
 {
   MOZ_COUNT_CTOR(AppleVDADecoder);
   // TODO: Verify aConfig.mime_type.
 
   mExtraData = aConfig.mExtraData;
   mMaxRefFrames = 4;
   // Retrieve video dimensions from H264 SPS NAL.
   mp4_demuxer::SPSData spsdata;
@@ -242,51 +243,111 @@ AppleVDADecoder::ClearReorderedFrames()
   }
 }
 
 // Copy and return a decoded frame.
 nsresult
 AppleVDADecoder::OutputFrame(CVPixelBufferRef aImage,
                              nsAutoPtr<AppleVDADecoder::AppleFrameRef> aFrameRef)
 {
-  IOSurfacePtr surface = MacIOSurfaceLib::CVPixelBufferGetIOSurface(aImage);
-  MOZ_ASSERT(surface, "Decoder didn't return an IOSurface backed buffer");
-
   LOG("mp4 output frame %lld dts %lld pts %lld duration %lld us%s",
-    aFrameRef->byte_offset,
-    aFrameRef->decode_timestamp,
-    aFrameRef->composition_timestamp,
-    aFrameRef->duration,
-    aFrameRef->is_sync_point ? " keyframe" : ""
+      aFrameRef->byte_offset,
+      aFrameRef->decode_timestamp,
+      aFrameRef->composition_timestamp,
+      aFrameRef->duration,
+      aFrameRef->is_sync_point ? " keyframe" : ""
   );
 
-  nsRefPtr<MacIOSurface> macSurface = new MacIOSurface(surface);
+  // Where our resulting image will end up.
+  nsRefPtr<VideoData> data;
   // Bounds.
   VideoInfo info;
   info.mDisplay = nsIntSize(mDisplayWidth, mDisplayHeight);
   gfx::IntRect visible = gfx::IntRect(0,
                                       0,
                                       mPictureWidth,
                                       mPictureHeight);
 
-  nsRefPtr<layers::Image> image =
-    mImageContainer->CreateImage(ImageFormat::MAC_IOSURFACE);
-  layers::MacIOSurfaceImage* videoImage =
-    static_cast<layers::MacIOSurfaceImage*>(image.get());
-  videoImage->SetSurface(macSurface);
+  if (mUseSoftwareImages) {
+    size_t width = CVPixelBufferGetWidth(aImage);
+    size_t height = CVPixelBufferGetHeight(aImage);
+    DebugOnly<size_t> planes = CVPixelBufferGetPlaneCount(aImage);
+    MOZ_ASSERT(planes == 2, "Likely not NV12 format and it must be.");
+
+    VideoData::YCbCrBuffer buffer;
 
-  nsRefPtr<VideoData> data;
-  data = VideoData::CreateFromImage(info,
-                                    mImageContainer,
-                                    aFrameRef->byte_offset,
-                                    aFrameRef->composition_timestamp,
-                                    aFrameRef->duration, image.forget(),
-                                    aFrameRef->is_sync_point,
-                                    aFrameRef->decode_timestamp,
-                                    visible);
+    // Lock the returned image data.
+    CVReturn rv = CVPixelBufferLockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly);
+    if (rv != kCVReturnSuccess) {
+      NS_ERROR("error locking pixel data");
+      mCallback->Error();
+      return NS_ERROR_FAILURE;
+    }
+    // Y plane.
+    buffer.mPlanes[0].mData =
+      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 0));
+    buffer.mPlanes[0].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 0);
+    buffer.mPlanes[0].mWidth = width;
+    buffer.mPlanes[0].mHeight = height;
+    buffer.mPlanes[0].mOffset = 0;
+    buffer.mPlanes[0].mSkip = 0;
+    // Cb plane.
+    buffer.mPlanes[1].mData =
+      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 1));
+    buffer.mPlanes[1].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 1);
+    buffer.mPlanes[1].mWidth = (width+1) / 2;
+    buffer.mPlanes[1].mHeight = (height+1) / 2;
+    buffer.mPlanes[1].mOffset = 0;
+    buffer.mPlanes[1].mSkip = 1;
+    // Cr plane.
+    buffer.mPlanes[2].mData =
+      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 1));
+    buffer.mPlanes[2].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 1);
+    buffer.mPlanes[2].mWidth = (width+1) / 2;
+    buffer.mPlanes[2].mHeight = (height+1) / 2;
+    buffer.mPlanes[2].mOffset = 1;
+    buffer.mPlanes[2].mSkip = 1;
+
+    // Copy the image data into our own format.
+    data =
+      VideoData::Create(info,
+                        mImageContainer,
+                        nullptr,
+                        aFrameRef->byte_offset,
+                        aFrameRef->composition_timestamp,
+                        aFrameRef->duration,
+                        buffer,
+                        aFrameRef->is_sync_point,
+                        aFrameRef->decode_timestamp,
+                        visible);
+    // Unlock the returned image data.
+    CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly);
+  } else {
+    IOSurfacePtr surface = MacIOSurfaceLib::CVPixelBufferGetIOSurface(aImage);
+    MOZ_ASSERT(surface, "Decoder didn't return an IOSurface backed buffer");
+
+    nsRefPtr<MacIOSurface> macSurface = new MacIOSurface(surface);
+
+    nsRefPtr<layers::Image> image =
+      mImageContainer->CreateImage(ImageFormat::MAC_IOSURFACE);
+    layers::MacIOSurfaceImage* videoImage =
+      static_cast<layers::MacIOSurfaceImage*>(image.get());
+    videoImage->SetSurface(macSurface);
+
+    data =
+      VideoData::CreateFromImage(info,
+                                 mImageContainer,
+                                 aFrameRef->byte_offset,
+                                 aFrameRef->composition_timestamp,
+                                 aFrameRef->duration,
+                                 image.forget(),
+                                 aFrameRef->is_sync_point,
+                                 aFrameRef->decode_timestamp,
+                                 visible);
+  }
 
   if (!data) {
     NS_ERROR("Couldn't create VideoData for frame");
     mCallback->Error();
     return NS_ERROR_FAILURE;
   }
 
   // Frames come out in DTS order but we need to output them
@@ -452,37 +513,54 @@ AppleVDADecoder::CreateDecoderSpecificat
                             ArrayLength(decoderKeys),
                             &kCFTypeDictionaryKeyCallBacks,
                             &kCFTypeDictionaryValueCallBacks);
 }
 
 CFDictionaryRef
 AppleVDADecoder::CreateOutputConfiguration()
 {
+  // Output format type:
+  SInt32 PixelFormatTypeValue = mUseSoftwareImages ?
+    kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_32BGRA;
+
+  AutoCFRelease<CFNumberRef> PixelFormatTypeNumber =
+    CFNumberCreate(kCFAllocatorDefault,
+                   kCFNumberSInt32Type,
+                   &PixelFormatTypeValue);
+
+  if (mUseSoftwareImages) {
+    const void* outputKeys[] = { kCVPixelBufferPixelFormatTypeKey };
+    const void* outputValues[] = { PixelFormatTypeNumber };
+    static_assert(ArrayLength(outputKeys) == ArrayLength(outputValues),
+                  "Non matching keys/values array size");
+
+    return CFDictionaryCreate(kCFAllocatorDefault,
+                              outputKeys,
+                              outputValues,
+                              ArrayLength(outputKeys),
+                              &kCFTypeDictionaryKeyCallBacks,
+                              &kCFTypeDictionaryValueCallBacks);
+  }
+
   // Construct IOSurface Properties
   const void* IOSurfaceKeys[] = { MacIOSurfaceLib::kPropIsGlobal };
   const void* IOSurfaceValues[] = { kCFBooleanTrue };
   static_assert(ArrayLength(IOSurfaceKeys) == ArrayLength(IOSurfaceValues),
                 "Non matching keys/values array size");
 
   // Contruct output configuration.
   AutoCFRelease<CFDictionaryRef> IOSurfaceProperties =
     CFDictionaryCreate(kCFAllocatorDefault,
                        IOSurfaceKeys,
                        IOSurfaceValues,
                        ArrayLength(IOSurfaceKeys),
                        &kCFTypeDictionaryKeyCallBacks,
                        &kCFTypeDictionaryValueCallBacks);
 
-  SInt32 PixelFormatTypeValue = kCVPixelFormatType_32BGRA;
-  AutoCFRelease<CFNumberRef> PixelFormatTypeNumber =
-    CFNumberCreate(kCFAllocatorDefault,
-                   kCFNumberSInt32Type,
-                   &PixelFormatTypeValue);
-
   const void* outputKeys[] = { kCVPixelBufferIOSurfacePropertiesKey,
                                kCVPixelBufferPixelFormatTypeKey,
                                kCVPixelBufferOpenGLCompatibilityKey };
   const void* outputValues[] = { IOSurfaceProperties,
                                  PixelFormatTypeNumber,
                                  kCFBooleanTrue };
   static_assert(ArrayLength(outputKeys) == ArrayLength(outputValues),
                 "Non matching keys/values array size");
--- a/dom/media/platforms/apple/AppleVDADecoder.h
+++ b/dom/media/platforms/apple/AppleVDADecoder.h
@@ -94,20 +94,21 @@ public:
   MediaDataDecoderCallback* mCallback;
   nsRefPtr<layers::ImageContainer> mImageContainer;
   ReorderQueue mReorderQueue;
   uint32_t mPictureWidth;
   uint32_t mPictureHeight;
   uint32_t mDisplayWidth;
   uint32_t mDisplayHeight;
   uint32_t mMaxRefFrames;
+  bool mUseSoftwareImages;
+  bool mIs106;
 
 private:
   VDADecoder mDecoder;
-  bool mIs106;
 
   // Method to pass a frame to VideoToolbox for decoding.
   nsresult SubmitFrame(MediaRawData* aSample);
   // Method to set up the decompression session.
   nsresult InitializeSession();
   CFDictionaryRef CreateDecoderSpecification();
 };
 
--- a/dom/media/platforms/apple/AppleVTDecoder.cpp
+++ b/dom/media/platforms/apple/AppleVTDecoder.cpp
@@ -7,17 +7,16 @@
 #include <CoreFoundation/CFString.h>
 
 #include "AppleCMLinker.h"
 #include "AppleUtils.h"
 #include "AppleVTDecoder.h"
 #include "AppleVTLinker.h"
 #include "mp4_demuxer/H264.h"
 #include "MediaData.h"
-#include "MacIOSurfaceImage.h"
 #include "mozilla/ArrayUtils.h"
 #include "nsAutoPtr.h"
 #include "nsThreadUtils.h"
 #include "mozilla/Logging.h"
 #include "VideoUtils.h"
 #include "gfxPlatform.h"
 
 PRLogModuleInfo* GetAppleMediaLog();
@@ -345,33 +344,29 @@ AppleVTDecoder::CreateDecoderExtensions(
                        atomsValue,
                        ArrayLength(atomsKey),
                        &kCFTypeDictionaryKeyCallBacks,
                        &kCFTypeDictionaryValueCallBacks);
 
   const void* extensionKeys[] =
     { kCVImageBufferChromaLocationBottomFieldKey,
       kCVImageBufferChromaLocationTopFieldKey,
-      AppleCMLinker::skPropExtensionAtoms,
-      AppleCMLinker::skPropFullRangeVideo /* Not defined in 10.6 */ };
+      AppleCMLinker::skPropExtensionAtoms };
 
   const void* extensionValues[] =
     { kCVImageBufferChromaLocation_Left,
       kCVImageBufferChromaLocation_Left,
-      atoms,
-      kCFBooleanTrue };
+      atoms };
   static_assert(ArrayLength(extensionKeys) == ArrayLength(extensionValues),
                 "Non matching keys/values array size");
 
   return CFDictionaryCreate(kCFAllocatorDefault,
                             extensionKeys,
                             extensionValues,
-                            AppleCMLinker::skPropFullRangeVideo ?
-                              ArrayLength(extensionKeys) :
-                              ArrayLength(extensionKeys) - 1,
+                            ArrayLength(extensionKeys),
                             &kCFTypeDictionaryKeyCallBacks,
                             &kCFTypeDictionaryValueCallBacks);
 }
 
 CFDictionaryRef
 AppleVTDecoder::CreateDecoderSpecification()
 {
   if (!AppleVTLinker::skPropEnableHWAccel) {