Bug 938034 - Add new GonkCameraImage image type. r=roc
☠☠ backed out by 8176e4ba6f4c ☠ ☠
authorAlfredo Yang <ayang@mozilla.com>
Mon, 15 Dec 2014 01:01:00 -0500
changeset 219775 bfe4f2eb91c57a4c0bfb0139a25d88010fab97d9
parent 219774 e4aed7beca31aafeda5550a06ba8813c281f358e
child 219776 4dabf09c2e196ad745573e250424b8b9dd421572
push id10419
push usercbook@mozilla.com
push dateTue, 16 Dec 2014 12:45:27 +0000
treeherderfx-team@ec87657146eb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersroc
bugs938034
milestone37.0a1
Bug 938034 - Add new GonkCameraImage image type. r=roc
dom/media/webrtc/GonkCameraImage.cpp
dom/media/webrtc/GonkCameraImage.h
dom/media/webrtc/moz.build
gfx/layers/ImageContainer.cpp
gfx/layers/ImageContainer.h
gfx/layers/ImageTypes.h
gfx/layers/client/ImageClient.cpp
new file mode 100644
--- /dev/null
+++ b/dom/media/webrtc/GonkCameraImage.cpp
@@ -0,0 +1,81 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "GonkCameraImage.h"
+#include "stagefright/MediaBuffer.h"
+
+namespace mozilla {
+
+GonkCameraImage::GonkCameraImage(layers::BufferRecycleBin* aRecycleBin)
+  : PlanarYCbCrImage(aRecycleBin)
+  , mMonitor("GonkCameraImage.Monitor")
+  , mMediaBuffer(nullptr)
+  , mThread(nullptr)
+{
+  mFormat = ImageFormat::GONK_CAMERA_IMAGE;
+}
+
+GonkCameraImage::~GonkCameraImage()
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+  // mMediaBuffer must be cleared before destructor.
+  MOZ_ASSERT(mMediaBuffer == nullptr);
+}
+
+nsresult
+GonkCameraImage::GetBuffer(android::MediaBuffer** aBuffer)
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+
+  if (!mMediaBuffer) {
+    return NS_ERROR_FAILURE;
+  }
+
+  MOZ_ASSERT(NS_GetCurrentThread() == mThread);
+
+  *aBuffer = mMediaBuffer;
+  mMediaBuffer->add_ref();
+
+  return NS_OK;
+}
+
+bool
+GonkCameraImage::HasMediaBuffer()
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+  return mMediaBuffer != nullptr;
+}
+
+nsresult
+GonkCameraImage::SetBuffer(android::MediaBuffer* aBuffer)
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+  MOZ_ASSERT(!mMediaBuffer);
+
+  mMediaBuffer = aBuffer;
+  mMediaBuffer->add_ref();
+  mThread = NS_GetCurrentThread();
+
+  return NS_OK;
+}
+
+nsresult
+GonkCameraImage::ClearBuffer()
+{
+  ReentrantMonitorAutoEnter mon(mMonitor);
+
+  if (mMediaBuffer) {
+    MOZ_ASSERT(NS_GetCurrentThread() == mThread);
+    mMediaBuffer->release();
+    mMediaBuffer = nullptr;
+    mThread = nullptr;
+  }
+  return NS_OK;
+}
+
+} // namespace mozilla
+
+
new file mode 100644
--- /dev/null
+++ b/dom/media/webrtc/GonkCameraImage.h
@@ -0,0 +1,74 @@
+/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef GONKCAMERAIMAGE_H
+#define GONKCAMERAIMAGE_H
+
+#include "mozilla/ReentrantMonitor.h"
+#include "ImageLayers.h"
+#include "ImageContainer.h"
+
+namespace android {
+class MOZ_EXPORT MediaBuffer;
+}
+
+namespace mozilla {
+
+/**
+ * GonkCameraImage has two parts. One is preview image which will be saved in
+ * PlanarYCbCrImage, another kind is the MediaBuffer keeps in mMediaBuffer
+ * which is from gonk camera recording callback. The data in MediaBuffer is Gonk
+ * shared memory based on android binder (IMemory), the actual format in IMemory
+ * is platform dependent.
+ * This instance is created in MediaEngine when the preview image arrives.
+ * The MediaBuffer is attached to the current created GonkCameraImage via SetBuffer().
+ * After sending this image to MediaStreamGraph by AppendToTrack(), ClearBuffer()
+ * must be called to clear MediaBuffer to avoid MediaBuffer be kept in MSG thread.
+ * The reason to keep MediaBuffer be accessed from MSG thread is MediaBuffer is
+ * limited resource and it could cause frame rate jitter if MediaBuffer stay too
+ * long in other threads.
+ * So there will be 3 threads to accessed this class. First is camera preview
+ * thread which creates an instance of this class and initialize the preview
+ * image in the base class PlanarYCbCrImage. Second is the camera recording
+ * thread which attaches MediaBuffer and sends this image to MediaStreamDirectListener.
+ * Third is the MSG thread via NotifyPull, the image should have preview image
+ * only in NotifyPull.
+ *
+ * Note: SetBuffer() and GetBuffer() should be called from the same thread. It
+ *       is forbidden to call GetBuffer() from other threads.
+ */
+class GonkCameraImage : public layers::PlanarYCbCrImage
+{
+public:
+  GonkCameraImage(layers::BufferRecycleBin* aRecycleBin);
+
+  // The returned aBuffer has called aBuffer->add_ref() already, so it is caller's
+  // duty to release aBuffer. It should be called from the same thread which
+  // called SetBuffer().
+  nsresult GetBuffer(android::MediaBuffer** aBuffer);
+
+  // Set MediaBuffer to image. It is caller's responsibility to call ClearBuffer()
+  // after the MediaBuffer is sent via MediaStreamGraph.
+  nsresult SetBuffer(android::MediaBuffer* aBuffer);
+
+  // It should be called from the same thread which called SetBuffer().
+  nsresult ClearBuffer();
+
+  bool HasMediaBuffer();
+
+protected:
+  virtual ~GonkCameraImage();
+
+  // mMonitor protects mMediaBuffer and mThread.
+  ReentrantMonitor mMonitor;
+  android::MediaBuffer* mMediaBuffer;
+  // Check if current thread is the same one which called SetBuffer().
+  // It doesn't need to hold reference count.
+  DebugOnly<nsIThread*> mThread;
+};
+
+} // namespace mozilla
+
+#endif /* GONKCAMERAIMAGE_H */
--- a/dom/media/webrtc/moz.build
+++ b/dom/media/webrtc/moz.build
@@ -31,18 +31,22 @@ if CONFIG['MOZ_WEBRTC']:
         '/dom/camera',
         '/media/libyuv/include',
         '/media/webrtc/signaling/src/common',
         '/media/webrtc/signaling/src/common/browser_logging',
         '/media/webrtc/trunk',
     ]
     # Gonk camera source.
     if CONFIG['MOZ_B2G_CAMERA']:
-        EXPORTS += ['MediaEngineGonkVideoSource.h']
+        EXPORTS += [
+            'GonkCameraImage.h',
+            'MediaEngineGonkVideoSource.h',
+        ]
         UNIFIED_SOURCES += [
+            'GonkCameraImage.cpp',
             'MediaEngineGonkVideoSource.cpp',
         ]
 
 XPIDL_SOURCES += [
     'nsITabSource.idl'
 ]
 
 UNIFIED_SOURCES += [
--- a/gfx/layers/ImageContainer.cpp
+++ b/gfx/layers/ImageContainer.cpp
@@ -14,16 +14,17 @@
 #include "mozilla/ipc/CrossProcessMutex.h"  // for CrossProcessMutex, etc
 #include "mozilla/layers/CompositorTypes.h"
 #include "mozilla/layers/ImageBridgeChild.h"  // for ImageBridgeChild
 #include "mozilla/layers/ImageClient.h"  // for ImageClient
 #include "nsISupportsUtils.h"           // for NS_IF_ADDREF
 #include "YCbCrUtils.h"                 // for YCbCr conversions
 #ifdef MOZ_WIDGET_GONK
 #include "GrallocImages.h"
+#include "GonkCameraImage.h"
 #endif
 #include "gfx2DGlue.h"
 #include "mozilla/gfx/2D.h"
 
 #ifdef XP_MACOSX
 #include "mozilla/gfx/QuartzSupport.h"
 #include "MacIOSurfaceImage.h"
 #endif
@@ -55,16 +56,20 @@ ImageFactory::CreateImage(ImageFormat aF
   if (aFormat == ImageFormat::GRALLOC_PLANAR_YCBCR) {
     img = new GrallocImage();
     return img.forget();
   }
   if (aFormat == ImageFormat::OVERLAY_IMAGE) {
     img = new OverlayImage();
     return img.forget();
   }
+  if (aFormat == ImageFormat::GONK_CAMERA_IMAGE) {
+    img = new GonkCameraImage(aRecycleBin);
+    return img.forget();
+  }
 #endif
   if (aFormat == ImageFormat::PLANAR_YCBCR) {
     img = new PlanarYCbCrImage(aRecycleBin);
     return img.forget();
   }
   if (aFormat == ImageFormat::CAIRO_SURFACE) {
     img = new CairoImage();
     return img.forget();
--- a/gfx/layers/ImageContainer.h
+++ b/gfx/layers/ImageContainer.h
@@ -100,16 +100,17 @@ class CrossProcessMutex;
 namespace layers {
 
 class ImageClient;
 class SharedPlanarYCbCrImage;
 class TextureClient;
 class CompositableClient;
 class CompositableForwarder;
 class SurfaceDescriptor;
+class PlanarYCbCrImage;
 
 struct ImageBackendData
 {
   virtual ~ImageBackendData() {}
 
 protected:
   ImageBackendData() {}
 };
@@ -162,16 +163,21 @@ public:
 
   int32_t GetSerial() { return mSerial; }
 
   void MarkSent() { mSent = true; }
   bool IsSentToCompositor() { return mSent; }
 
   virtual TemporaryRef<gfx::SourceSurface> GetAsSourceSurface() = 0;
 
+  virtual PlanarYCbCrImage* AsPlanarYCbCrImage()
+  {
+    return nullptr;
+  }
+
 protected:
   Image(void* aImplData, ImageFormat aFormat) :
     mImplData(aImplData),
     mSerial(++sSerialCounter),
     mFormat(aFormat),
     mSent(false)
   {}
 
@@ -746,16 +752,21 @@ public:
   virtual SharedPlanarYCbCrImage *AsSharedPlanarYCbCrImage() { return nullptr; }
 
   virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
   }
 
   virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
 
+  virtual PlanarYCbCrImage* AsPlanarYCbCrImage() MOZ_OVERRIDE
+  {
+    return this;
+  }
+
 protected:
   /**
    * Make a copy of the YCbCr data into local storage.
    *
    * @param aData           Input image data.
    */
   void CopyData(const Data& aData);
 
--- a/gfx/layers/ImageTypes.h
+++ b/gfx/layers/ImageTypes.h
@@ -21,16 +21,26 @@ MOZ_BEGIN_ENUM_CLASS(ImageFormat)
   /**
    * The GRALLOC_PLANAR_YCBCR format creates a GrallocImage, a subtype of
    * PlanarYCbCrImage. It takes a PlanarYCbCrImage data or the raw gralloc
    * data and can be used as a texture by Gonk backend directly.
    */
   GRALLOC_PLANAR_YCBCR,
 
   /**
+   * The GONK_CAMERA_IMAGE format creates a GonkCameraImage, which contains two
+   * parts. One is PlanarYCbCr image for preview image. Another one is
+   * MediaBuffer from Gonk recording image. The preview image can be rendered in
+   * a layer for display. And the MediaBuffer will be used in component like OMX
+   * encoder. It is for GUM to support preview and recording image on Gonk
+   * camera.
+   */
+  GONK_CAMERA_IMAGE,
+
+  /**
    * The SHARED_RGB format creates a SharedRGBImage, which stores RGB data in
    * shared memory. Some Android hardware video decoders require this format.
    * Currently only used on Android.
    */
   SHARED_RGB,
 
   /**
    * The CAIRO_SURFACE format creates a CairoImage. All backends should
--- a/gfx/layers/client/ImageClient.cpp
+++ b/gfx/layers/client/ImageClient.cpp
@@ -160,19 +160,18 @@ ImageClientSingle::UpdateImage(ImageCont
     autoRemoveTexture.mTexture = mFrontBuffer;
     mFrontBuffer = nullptr;
   }
 
   if (!texture) {
     // Slow path, we should not be hitting it very often and if we do it means
     // we are using an Image class that is not backed by textureClient and we
     // should fix it.
-    if (image->GetFormat() == ImageFormat::PLANAR_YCBCR) {
-      PlanarYCbCrImage* ycbcr = static_cast<PlanarYCbCrImage*>(image);
-      const PlanarYCbCrData* data = ycbcr->GetData();
+    if (image->AsPlanarYCbCrImage()) {
+      const PlanarYCbCrData* data = image->AsPlanarYCbCrImage()->GetData();
       if (!data) {
         return false;
       }
       texture = TextureClient::CreateForYCbCr(GetForwarder(),
         data->mYSize, data->mCbCrSize, data->mStereoMode,
         TextureFlags::DEFAULT | mTextureFlags
       );
       if (!texture || !texture->Lock(OpenMode::OPEN_WRITE_ONLY)) {