Bug Bug 773440 - Remove unnecessary frame copies with async-video by stroring video frames in shared memory earlier in the pipeline. r=roc
authorNicolas Silva <nical.bugzilla@gmail.com>
Mon, 05 Nov 2012 11:38:03 +0100
changeset 112305 35ba50a6a97e3df90392394bc177fa6e2941ce90
parent 112301 60c78a559a84441c175506792d9f977a8f7a9113
child 112306 dc1d29a3f2a7045f905a1b56f5de987028fb4061
push id23812
push useremorley@mozilla.com
push dateTue, 06 Nov 2012 14:01:34 +0000
treeherdermozilla-central@f4aeed115e54 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersroc
bugs773440
milestone19.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug Bug 773440 - Remove unnecessary frame copies with async-video by stroring video frames in shared memory earlier in the pipeline. r=roc
gfx/layers/ImageContainer.cpp
gfx/layers/ImageContainer.h
gfx/layers/ipc/ImageContainerChild.cpp
gfx/layers/ipc/ImageContainerChild.h
--- a/gfx/layers/ImageContainer.cpp
+++ b/gfx/layers/ImageContainer.cpp
@@ -139,16 +139,22 @@ ImageContainer::~ImageContainer()
   }
 }
 
 already_AddRefed<Image>
 ImageContainer::CreateImage(const ImageFormat *aFormats,
                             uint32_t aNumFormats)
 {
   ReentrantMonitorAutoEnter mon(mReentrantMonitor);
+  if (mImageContainerChild) {
+    nsRefPtr<Image> img = mImageContainerChild->CreateImage();
+    if (img) {
+      return img.forget();
+    }
+  }
   return mImageFactory->CreateImage(aFormats, aNumFormats, mScaleHint, mRecycleBin);
 }
 
 void 
 ImageContainer::SetCurrentImageInternal(Image *aImage)
 {
   ReentrantMonitorAutoEnter mon(mReentrantMonitor);
 
--- a/gfx/layers/ImageContainer.h
+++ b/gfx/layers/ImageContainer.h
@@ -34,16 +34,17 @@ namespace mozilla {
 class CrossProcessMutex;
 namespace ipc {
 class Shmem;
 }
     
 namespace layers {
 
 class ImageContainerChild;
+class SharedPlanarYCbCrImage;
 
 struct ImageBackendData
 {
   virtual ~ImageBackendData() {}
 
 protected:
   ImageBackendData() {}
 };
@@ -77,29 +78,34 @@ public:
 
   ImageBackendData* GetBackendData(LayersBackend aBackend)
   { return mBackendData[aBackend]; }
   void SetBackendData(LayersBackend aBackend, ImageBackendData* aData)
   { mBackendData[aBackend] = aData; }
 
   int32_t GetSerial() { return mSerial; }
 
+  void MarkSent() { mSent = true; }
+  bool IsSentToCompositor() { return mSent; }
+
 protected:
   Image(void* aImplData, ImageFormat aFormat) :
     mImplData(aImplData),
     mSerial(PR_ATOMIC_INCREMENT(&sSerialCounter)),
-    mFormat(aFormat)
+    mFormat(aFormat),
+    mSent(false)
   {}
 
   nsAutoPtr<ImageBackendData> mBackendData[mozilla::layers::LAYERS_LAST];
 
   void* mImplData;
   int32_t mSerial;
   ImageFormat mFormat;
   static int32_t sSerialCounter;
+  bool mSent;
 };
 
 /**
  * A RecycleBin is owned by an ImageContainer. We store buffers in it that we
  * want to recycle from one image to the next.It's a separate object from 
  * ImageContainer because images need to store a strong ref to their RecycleBin
  * and we must avoid creating a reference loop between an ImageContainer and
  * its active image.
@@ -691,16 +697,18 @@ public:
   virtual uint32_t GetDataSize() { return mBufferSize; }
 
   virtual bool IsValid() { return !!mBufferSize; }
 
   virtual gfxIntSize GetSize() { return mSize; }
 
   PlanarYCbCrImage(BufferRecycleBin *aRecycleBin);
 
+  virtual SharedPlanarYCbCrImage *AsSharedPlanarYCbCrImage() { return nullptr; }
+
 protected:
   /**
    * Make a copy of the YCbCr data into local storage.
    *
    * @param aData           Input image data.
    */
   void CopyData(const Data& aData);
 
--- a/gfx/layers/ipc/ImageContainerChild.cpp
+++ b/gfx/layers/ipc/ImageContainerChild.cpp
@@ -8,16 +8,17 @@
 #include "gfxSharedImageSurface.h"
 #include "ShadowLayers.h"
 #include "mozilla/layers/PLayers.h"
 #include "mozilla/layers/SharedImageUtils.h"
 #include "ImageContainer.h"
 #include "GonkIOSurfaceImage.h"
 #include "GrallocImages.h"
 #include "mozilla/layers/ShmemYCbCrImage.h"
+#include "mozilla/ReentrantMonitor.h"
 
 using namespace mozilla::ipc;
 
 namespace mozilla {
 namespace layers {
 
 /*
  * - POOL_MAX_SHARED_IMAGES is the maximum number number of shared images to
@@ -29,17 +30,16 @@ namespace layers {
  * dealocated by this ImageContainerChild. What can happen is that the compositor
  * hangs for a moment, while the ImageBridgeChild keep sending images. In such a 
  * scenario the compositor is not sending back shared images so the 
  * ImageContinerChild allocates new ones, and if the compositor hangs for too 
  * long, we can run out of shared memory. MAX_ACTIVE_SHARED_IMAGES is there to
  * throttle that. So when the child side wants to allocate a new shared image 
  * but is already at its maximum of active shared images, it just discards the
  * image (which is therefore not allocated and not sent to the compositor).
- * (see ImageToSharedImage)
  *
  * The values for the two constants are arbitrary and should be tweaked if it 
  * happens that we run into shared memory problems.
  */
 static const unsigned int POOL_MAX_SHARED_IMAGES = 5;
 static const unsigned int MAX_ACTIVE_SHARED_IMAGES = 10;
 
 ImageContainerChild::ImageContainerChild()
@@ -119,32 +119,16 @@ void ImageContainerChild::DestroySharedI
 {
   NS_ABORT_IF_FALSE(InImageBridgeChildThread(),
                     "Should be in ImageBridgeChild thread.");
 
   --mActiveImageCount;
   DeallocSharedImageData(this, aImage);
 }
 
-SharedImage* ImageContainerChild::AsSharedImage(Image* aImage)
-{
-#ifdef MOZ_WIDGET_GONK
-  if (aImage->GetFormat() == GONK_IO_SURFACE) {
-    GonkIOSurfaceImage* gonkImage = static_cast<GonkIOSurfaceImage*>(aImage);
-    SharedImage* result = new SharedImage(gonkImage->GetSurfaceDescriptor());
-    return result;
-  } else if (aImage->GetFormat() == GRALLOC_PLANAR_YCBCR) {
-    GrallocPlanarYCbCrImage* GrallocImage = static_cast<GrallocPlanarYCbCrImage*>(aImage);
-    SharedImage* result = new SharedImage(GrallocImage->GetSurfaceDescriptor());
-    return result;
-  }
-#endif
-  return nullptr; // XXX: bug 773440
-}
-
 bool ImageContainerChild::CopyDataIntoSharedImage(Image* src, SharedImage* dest)
 {
   if ((src->GetFormat() == PLANAR_YCBCR) && 
       (dest->type() == SharedImage::TYCbCrImage)) {
     PlanarYCbCrImage *planarYCbCrImage = static_cast<PlanarYCbCrImage*>(src);
     const PlanarYCbCrImage::Data *data = planarYCbCrImage->GetData();
     NS_ASSERTION(data, "Must be able to retrieve yuv data from image!");
     YCbCrImage& yuv = dest->get_YCbCrImage();
@@ -200,16 +184,41 @@ SharedImage* ImageContainerChild::Alloca
     ++mActiveImageCount;
     return new SharedImage(YCbCrImage(shmem, 0, data->GetPictureRect()));
   } else {
     NS_RUNTIMEABORT("TODO: Only YCbCrImage is supported here right now.");
   }
   return nullptr;
 }
 
+void ImageContainerChild::RecycleSharedImageNow(SharedImage* aImage)
+{
+  NS_ABORT_IF_FALSE(InImageBridgeChildThread(),"Must be in the ImageBridgeChild Thread.");
+
+  if (mStop || !AddSharedImageToPool(aImage)) {
+    DestroySharedImage(*aImage);
+    delete aImage;
+  }
+}
+
+void ImageContainerChild::RecycleSharedImage(SharedImage* aImage)
+{
+  if (!aImage) {
+    return;
+  }
+  if (InImageBridgeChildThread()) {
+    RecycleSharedImageNow(aImage);
+    return;
+  }
+  GetMessageLoop()->PostTask(FROM_HERE,
+                             NewRunnableMethod(this,
+                                               &ImageContainerChild::RecycleSharedImageNow,
+                                               aImage));
+}
+
 bool ImageContainerChild::AddSharedImageToPool(SharedImage* img)
 {
   NS_ABORT_IF_FALSE(InImageBridgeChildThread(), 
                     "AddSharedImageToPool must be called in the ImageBridgeChild thread");
   if (mStop) {
     return false;
   }
 
@@ -284,32 +293,37 @@ void ImageContainerChild::SendImageNow(I
 {
   NS_ABORT_IF_FALSE(InImageBridgeChildThread(),
                     "Should be in ImageBridgeChild thread.");
 
   if (mStop) {
     return;
   }
 
+  if (aImage->IsSentToCompositor()) {
+    return;
+  }
+
   bool needsCopy = false;
   // If the image can be converted to a shared image, no need to do a copy.
   SharedImage* img = AsSharedImage(aImage);
   if (!img) {
     needsCopy = true;
     // Try to get a compatible shared image from the pool
     img = GetSharedImageFor(aImage);
     if (!img && mActiveImageCount < (int)MAX_ACTIVE_SHARED_IMAGES) {
       // If no shared image available, allocate a new one
       img = AllocateSharedImageFor(aImage);
     }
   }
 
   if (img && (!needsCopy || CopyDataIntoSharedImage(aImage, img))) {
     // Keep a reference to the image we sent to compositor to maintain a
     // correct reference count.
+    aImage->MarkSent();
     mImageQueue.AppendElement(aImage);
     SendPublishImage(*img);
   } else {
     NS_WARNING("Failed to send an image to the compositor");
   }
   delete img;
   return;
 }
@@ -376,11 +390,234 @@ void ImageContainerChild::DispatchDestro
     return;
   }
   mDispatchedDestroy = true;
   AddRef(); // corresponds to the Release in DestroyNow
   GetMessageLoop()->PostTask(FROM_HERE, 
                     NewRunnableMethod(this, &ImageContainerChild::DestroyNow));
 }
 
+// We can't pass more than 6 parameters to a 'NewRunableFunction' so some
+// parameters are stored in a struct passed by pointer
+struct CreateShmemParams
+{
+  ImageContainerChild* mProtocol;
+  size_t mBufSize;
+  SharedMemory::SharedMemoryType mType;
+  ipc::Shmem* mShmem;
+  bool mResult;
+};
+
+static void AllocUnsafeShmemNow(CreateShmemParams* aParams,
+                                ReentrantMonitor* aBarrier,
+                                bool* aDone)
+{
+  ReentrantMonitorAutoEnter autoBarrier(*aBarrier);
+  aParams->mResult = aParams->mProtocol->AllocUnsafeShmem(aParams->mBufSize,
+                                                          aParams->mType,
+                                                          aParams->mShmem);
+*aDone = true;
+  aBarrier->NotifyAll();
+}
+
+
+bool ImageContainerChild::AllocUnsafeShmemSync(size_t aBufSize,
+                                               SharedMemory::SharedMemoryType aType,
+                                               ipc::Shmem* aShmem)
+{
+  if (mStop) {
+    return false;
+  }
+  if (InImageBridgeChildThread()) {
+    AllocUnsafeShmem(aBufSize, aType, aShmem);
+  }
+  ReentrantMonitor barrier("ImageContainerChild::AllocUnsafeShmemSync");
+  ReentrantMonitorAutoEnter autoBarrier(barrier);
+
+  CreateShmemParams p = {
+    this,
+    aBufSize,
+    aType,
+    aShmem,
+    false,
+  };
+
+  bool done = false;
+  GetMessageLoop()->PostTask(FROM_HERE,
+                 NewRunnableFunction(&AllocUnsafeShmemNow,
+                                     &p,
+                                     &barrier, &done));
+  while (!done) {
+    barrier.Wait();
+  }
+
+  return p.mResult;
+}
+
+static void DeallocShmemNow(ImageContainerChild* aProtocol, ipc::Shmem aShmem)
+{
+  aProtocol->DeallocShmem(aShmem);
+}
+
+void ImageContainerChild::DeallocShmemAsync(ipc::Shmem& aShmem)
+{
+  if (mStop) {
+    return;
+  }
+  GetMessageLoop()->PostTask(FROM_HERE,
+                             NewRunnableFunction(&DeallocShmemNow,
+                                                 this,
+                                                 aShmem));
+}
+
+class SharedPlanarYCbCrImage : public PlanarYCbCrImage
+{
+public:
+  SharedPlanarYCbCrImage(ImageContainerChild* aProtocol)
+  : PlanarYCbCrImage(nullptr),
+    mImageContainerChild(aProtocol), mAllocated(false) {}
+
+  ~SharedPlanarYCbCrImage() {
+    if (mAllocated) {
+      mImageContainerChild->RecycleSharedImage(ToSharedImage());
+    }
+  }
+
+  virtual SharedPlanarYCbCrImage* AsSharedPlanarYCbCrImage() MOZ_OVERRIDE
+  {
+    return this;
+  }
+
+  virtual already_AddRefed<gfxASurface> GetAsSurface() MOZ_OVERRIDE
+  {
+    if (!mAllocated) {
+      NS_WARNING("Can't get as surface");
+      return nullptr;
+    }
+    return PlanarYCbCrImage::GetAsSurface();
+  }
+
+  virtual void SetData(const PlanarYCbCrImage::Data& aData) MOZ_OVERRIDE
+  {
+    // If mShmem has not been allocated (through Allocate(aData)), allocate it.
+    // This code path is slower than the one used when Allocate has been called
+    // since it will trigger a full copy.
+    if (!mAllocated) {
+      Data data = aData;
+      if (!Allocate(data)) {
+        return;
+      }
+    }
+
+    // do not set mBuffer like in PlanarYCbCrImage because the later
+    // will try to manage this memory without knowing it belongs to a
+    // shmem.
+    mBufferSize = ShmemYCbCrImage::ComputeMinBufferSize(mData.mYSize,
+                                                        mData.mCbCrSize);
+    mSize = mData.mPicSize;
+
+    ShmemYCbCrImage shmImg(mShmem);
+
+    if (!shmImg.CopyData(aData.mYChannel, aData.mCbChannel, aData.mCrChannel,
+                         aData.mYSize, aData.mYStride,
+                         aData.mCbCrSize, aData.mCbCrStride)) {
+      NS_WARNING("Failed to copy image data!");
+    }
+    mData.mYChannel = shmImg.GetYData();
+    mData.mCbChannel = shmImg.GetCbData();
+    mData.mCrChannel = shmImg.GetCrData();
+  }
+
+  virtual bool Allocate(PlanarYCbCrImage::Data& aData)
+  {
+    NS_ABORT_IF_FALSE(!mAllocated, "This image already has allocated data");
+
+    SharedMemory::SharedMemoryType shmType = OptimalShmemType();
+    size_t size = ShmemYCbCrImage::ComputeMinBufferSize(aData.mYSize,
+                                                        aData.mCbCrSize);
+
+    if (!mImageContainerChild->AllocUnsafeShmemSync(size, shmType, &mShmem)) {
+      return false;
+    }
+    ShmemYCbCrImage::InitializeBufferInfo(mShmem.get<uint8_t>(),
+                                          aData.mYSize,
+                                          aData.mCbCrSize);
+    ShmemYCbCrImage shmImg(mShmem);
+    if (!shmImg.IsValid() || mShmem.Size<uint8_t>() < size) {
+      mImageContainerChild->DeallocShmemAsync(mShmem);
+      return false;
+    }
+
+    aData.mYChannel = shmImg.GetYData();
+    aData.mCbChannel = shmImg.GetCbData();
+    aData.mCrChannel = shmImg.GetCrData();
+
+    // copy some of aData's values in mData (most of them)
+    mData.mYChannel = aData.mYChannel;
+    mData.mCbChannel = aData.mCbChannel;
+    mData.mCrChannel = aData.mCrChannel;
+    mData.mYSize = aData.mYSize;
+    mData.mCbCrSize = aData.mCbCrSize;
+    mData.mPicX = aData.mPicX;
+    mData.mPicY = aData.mPicY;
+    mData.mPicSize = aData.mPicSize;
+    mData.mStereoMode = aData.mStereoMode;
+    // those members are not always equal to aData's, due to potentially different
+    // packing.
+    mData.mYSkip = 0;
+    mData.mCbSkip = 0;
+    mData.mCrSkip = 0;
+    mData.mYStride = mData.mYSize.width;
+    mData.mCbCrStride = mData.mCbCrSize.width;
+
+    mAllocated = true;
+    return true;
+  }
+
+  virtual bool IsValid() MOZ_OVERRIDE {
+    return mAllocated;
+  }
+
+  SharedImage* ToSharedImage() {
+    if (mAllocated) {
+      return new SharedImage(YCbCrImage(mShmem, 0, mData.GetPictureRect()));
+    }
+    return nullptr;
+  }
+
+private:
+  Shmem mShmem;
+  nsRefPtr<ImageContainerChild> mImageContainerChild;
+  bool mAllocated;
+};
+
+already_AddRefed<Image> ImageContainerChild::CreateImage()
+{
+  nsRefPtr<Image> img = new SharedPlanarYCbCrImage(this);
+  return img.forget();
+}
+
+SharedImage* ImageContainerChild::AsSharedImage(Image* aImage)
+{
+#ifdef MOZ_WIDGET_GONK
+  if (aImage->GetFormat() == GONK_IO_SURFACE) {
+    GonkIOSurfaceImage* gonkImage = static_cast<GonkIOSurfaceImage*>(aImage);
+    SharedImage* result = new SharedImage(gonkImage->GetSurfaceDescriptor());
+    return result;
+  } else if (aImage->GetFormat() == GRALLOC_PLANAR_YCBCR) {
+    GrallocPlanarYCbCrImage* GrallocImage = static_cast<GrallocPlanarYCbCrImage*>(aImage);
+    SharedImage* result = new SharedImage(GrallocImage->GetSurfaceDescriptor());
+    return result;
+  }
+#endif
+  if (aImage->GetFormat() == PLANAR_YCBCR) {
+    SharedPlanarYCbCrImage* sharedYCbCr
+      = static_cast<PlanarYCbCrImage*>(aImage)->AsSharedPlanarYCbCrImage();
+    if (sharedYCbCr) {
+      return sharedYCbCr->ToSharedImage();
+    }
+  }
+  return nullptr;
+}
+
 } // namespace
 } // namespace
 
--- a/gfx/layers/ipc/ImageContainerChild.h
+++ b/gfx/layers/ipc/ImageContainerChild.h
@@ -5,16 +5,17 @@
 
 #ifndef MOZILLA_GFX_IMAGECONTAINERCHILD_H
 #define MOZILLA_GFX_IMAGECONTAINERCHILD_H
 
 #include "mozilla/layers/PImageContainerChild.h"
 #include "mozilla/layers/ImageBridgeChild.h"
 
 namespace mozilla {
+class ReentrantMonitor;
 namespace layers {
 
 class ImageBridgeCopyAndSendTask;
 class ImageContainer;
 
 /**
  * ImageContainerChild is the content-side part of the ImageBridge IPDL protocol
  * acting at the ImageContainer level.
@@ -110,18 +111,50 @@ public:
    * Can be called on any thread.
    */
   void DispatchSetIdle();
 
   /**
    * Must be called on the ImageBridgeChild's thread.
    */
   void SetIdleNow();
-  
+
+  /**
+   * Can be called from any thread.
+   * deallocates or places aImage in a pool.
+   * If this method is not called on the ImageBridgeChild thread, a task is
+   * is dispatched and the recycling/deallocation happens asynchronously on
+   * the ImageBridgeChild thread.
+   */
+  void RecycleSharedImage(SharedImage* aImage);
+
+  /**
+   * Creates a YCbCrImage using shared memory to store its data.
+   */
+  already_AddRefed<Image> CreateImage();
+
+  /**
+   * Allocates an unsafe shmem.
+   * Unlike AllocUnsafeShmem, this method can be called from any thread.
+   * If the calling thread is not the ImageBridgeChild thread, this method will
+   * dispatch a synchronous call to AllocUnsafeShmem on the ImageBridgeChild
+   * thread meaning that the calling thread will be blocked until
+   * AllocUnsafeShmem returns on the ImageBridgeChild thread.
+   */
+  bool AllocUnsafeShmemSync(size_t aBufSize,
+                            SharedMemory::SharedMemoryType aType,
+                            ipc::Shmem* aShmem);
+
+  /**
+   * Dispatches a task on the ImageBridgeChild thread to deallocate a shmem.
+   */
+  void DeallocShmemAsync(ipc::Shmem& aShmem);
+
 protected:
+
   virtual PGrallocBufferChild*
   AllocPGrallocBuffer(const gfxIntSize&, const gfxContentType&,
                       MaybeMagicGrallocBufferHandle*) MOZ_OVERRIDE
 
   { return nullptr; }
 
   virtual bool
   DeallocPGrallocBuffer(PGrallocBufferChild* actor) MOZ_OVERRIDE
@@ -138,16 +171,21 @@ protected:
   void DestroyNow();
 
   inline void SetID(uint64_t id)
   {
     mImageContainerID = id;
   }
 
   /**
+   * Must be called on the ImageBirdgeChild thread. (See RecycleSharedImage)
+   */
+  void RecycleSharedImageNow(SharedImage* aImage);
+
+  /**
    * Deallocates a shared image's shared memory.
    * It is the caller's responsibility to delete the SharedImage itself.
    */
   void DestroySharedImage(const SharedImage& aSurface);
 
   /**
    * Each ImageContainerChild keeps a pool of SharedImages to avoid 
    * deallocationg/reallocating too often.
@@ -185,16 +223,17 @@ protected:
    */
   SharedImage* GetSharedImageFor(Image* aImage);
 
   /**
    * Allocates a SharedImage.
    * Returns nullptr in case of failure.
    * The returned image does not contain the image data, a copy still needs to
    * be done afterward (see CopyDataIntoSharedImage).
+   * Must be called on the ImageBridgeChild thread.
    */
   SharedImage* AllocateSharedImageFor(Image* aImage);
 
   /**
    * Called by ImageToSharedImage.
    */
   bool CopyDataIntoSharedImage(Image* src, SharedImage* dest);