Bug 1362303 - Don't use a Shmem we failed to allocate in nsContentUtils. r=Bas
authorNicolas Silva <nsilva@mozilla.com>
Mon, 15 May 2017 12:15:48 +0200
changeset 406542 54af3ab740d0723620f6e2b01d5c98e8f3a17b12
parent 406541 9ff22abe573688381dd2af3a9d1a85dd7a95c1c7
child 406543 046aa57cf9fe881a70779a288e67e69b6e5c1cf9
push id7391
push usermtabara@mozilla.com
push dateMon, 12 Jun 2017 13:08:53 +0000
treeherdermozilla-beta@2191d7f87e2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersBas
bugs1362303
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1362303 - Don't use a Shmem we failed to allocate in nsContentUtils. r=Bas
dom/base/nsContentUtils.cpp
dom/base/nsContentUtils.h
widget/nsDragServiceProxy.cpp
--- a/dom/base/nsContentUtils.cpp
+++ b/dom/base/nsContentUtils.cpp
@@ -8122,27 +8122,29 @@ nsContentUtils::TransferableToIPCTransfe
             }
             RefPtr<mozilla::gfx::DataSourceSurface> dataSurface =
               surface->GetDataSurface();
             if (!dataSurface) {
               continue;
             }
             size_t length;
             int32_t stride;
-            Shmem surfaceData;
             IShmemAllocator* allocator = aChild ? static_cast<IShmemAllocator*>(aChild)
                                                 : static_cast<IShmemAllocator*>(aParent);
-            GetSurfaceData(dataSurface, &length, &stride,
-                           allocator,
-                           &surfaceData);
+            Maybe<Shmem> surfaceData = GetSurfaceData(dataSurface, &length, &stride,
+                                                      allocator);
+
+            if (surfaceData.isNothing()) {
+              continue;
+            }
 
             IPCDataTransferItem* item = aIPCDataTransfer->items().AppendElement();
             item->flavor() = flavorStr;
             // Turn item->data() into an nsCString prior to accessing it.
-            item->data() = surfaceData;
+            item->data() = surfaceData.ref();
 
             IPCDataTransferImage& imageDetails = item->imageDetails();
             mozilla::gfx::IntSize size = dataSurface->GetSize();
             imageDetails.width() = size.width;
             imageDetails.height() = size.height;
             imageDetails.stride() = stride;
             imageDetails.format() = static_cast<uint8_t>(dataSurface->GetFormat());
 
@@ -8282,36 +8284,37 @@ struct GetSurfaceDataRawBuffer
     return ReturnType();
   }
 };
 
 // The type used for calling GetSurfaceData() that allocates and writes to
 // a shared memory buffer.
 struct GetSurfaceDataShmem
 {
-  using ReturnType = Shmem;
+  using ReturnType = Maybe<Shmem>;
   using BufferType = char*;
 
   explicit GetSurfaceDataShmem(IShmemAllocator* aAllocator)
     : mAllocator(aAllocator)
   { }
 
   ReturnType Allocate(size_t aSize)
   {
-    Shmem returnValue;
-    mAllocator->AllocShmem(aSize,
-                           SharedMemory::TYPE_BASIC,
-                           &returnValue);
-    return returnValue;
+    Shmem shmem;
+    if (!mAllocator->AllocShmem(aSize, SharedMemory::TYPE_BASIC, &shmem)) {
+      return Nothing();
+    }
+
+    return Some(shmem);
   }
 
   static BufferType
   GetBuffer(const ReturnType& aReturnValue)
   {
-    return aReturnValue.get<char>();
+    return aReturnValue.isSome() ? aReturnValue.ref().get<char>() : nullptr;
   }
 
   static ReturnType
   NullValue()
   {
     return ReturnType();
   }
 private:
@@ -8369,24 +8372,23 @@ GetSurfaceDataImpl(mozilla::gfx::DataSou
 mozilla::UniquePtr<char[]>
 nsContentUtils::GetSurfaceData(
   NotNull<mozilla::gfx::DataSourceSurface*> aSurface,
   size_t* aLength, int32_t* aStride)
 {
   return GetSurfaceDataImpl(aSurface, aLength, aStride);
 }
 
-void
+Maybe<Shmem>
 nsContentUtils::GetSurfaceData(mozilla::gfx::DataSourceSurface* aSurface,
                                size_t* aLength, int32_t* aStride,
-                               IShmemAllocator* aAllocator,
-                               Shmem *aOutShmem)
-{
-  *aOutShmem = GetSurfaceDataImpl(aSurface, aLength, aStride,
-                                  GetSurfaceDataShmem(aAllocator));
+                               IShmemAllocator* aAllocator)
+{
+  return GetSurfaceDataImpl(aSurface, aLength, aStride,
+                            GetSurfaceDataShmem(aAllocator));
 }
 
 mozilla::Modifiers
 nsContentUtils::GetWidgetModifiers(int32_t aModifiers)
 {
   Modifiers result = 0;
   if (aModifiers & nsIDOMWindowUtils::MODIFIER_SHIFT) {
     result |= mozilla::MODIFIER_SHIFT;
--- a/dom/base/nsContentUtils.h
+++ b/dom/base/nsContentUtils.h
@@ -30,16 +30,17 @@
 #include "Units.h"
 #include "mozilla/dom/AutocompleteInfoBinding.h"
 #include "mozilla/dom/BindingDeclarations.h" // For CallerType
 #include "mozilla/dom/ScriptSettings.h"
 #include "mozilla/FloatingPoint.h"
 #include "mozilla/net/ReferrerPolicy.h"
 #include "mozilla/Logging.h"
 #include "mozilla/NotNull.h"
+#include "mozilla/Maybe.h"
 #include "nsIContentPolicy.h"
 #include "nsIDocument.h"
 #include "nsPIDOMWindow.h"
 #include "nsRFPService.h"
 
 #if defined(XP_WIN)
 // Undefine LoadImage to prevent naming conflict with Windows.
 #undef LoadImage
@@ -2668,20 +2669,20 @@ public:
   static mozilla::UniquePtr<char[]> GetSurfaceData(
     mozilla::NotNull<mozilla::gfx::DataSourceSurface*> aSurface,
     size_t* aLength, int32_t* aStride);
 
   /*
    * Get the pixel data from the given source surface and fill it in Shmem.
    * The length and stride will be assigned from the surface.
    */
-  static void GetSurfaceData(mozilla::gfx::DataSourceSurface* aSurface,
-                             size_t* aLength, int32_t* aStride,
-                             mozilla::ipc::IShmemAllocator* aAlloc,
-                             mozilla::ipc::Shmem *aOutShmem);
+  static mozilla::Maybe<mozilla::ipc::Shmem>
+  GetSurfaceData(mozilla::gfx::DataSourceSurface* aSurface,
+                 size_t* aLength, int32_t* aStride,
+                 mozilla::ipc::IShmemAllocator* aAlloc);
 
   // Helpers shared by the implementations of nsContentUtils methods and
   // nsIDOMWindowUtils methods.
   static mozilla::Modifiers GetWidgetModifiers(int32_t aModifiers);
   static nsIWidget* GetWidget(nsIPresShell* aPresShell, nsPoint* aOffset);
   static int16_t GetButtonsFlagForButton(int32_t aButton);
   static mozilla::LayoutDeviceIntPoint ToWidgetPoint(const mozilla::CSSPoint& aPoint,
                                                      const nsPoint& aOffset,
--- a/widget/nsDragServiceProxy.cpp
+++ b/widget/nsDragServiceProxy.cpp
@@ -50,19 +50,26 @@ nsDragServiceProxy::InvokeDragSessionImp
     DrawDrag(mSourceNode, aRegion, mScreenPosition, &dragRect, &surface, &pc);
 
     if (surface) {
       RefPtr<mozilla::gfx::DataSourceSurface> dataSurface =
         surface->GetDataSurface();
       if (dataSurface) {
         size_t length;
         int32_t stride;
-        Shmem surfaceData;
-        nsContentUtils::GetSurfaceData(dataSurface, &length, &stride, child,
-                                       &surfaceData);
+        Maybe<Shmem> maybeShm = nsContentUtils::GetSurfaceData(dataSurface,
+                                                               &length,
+                                                               &stride,
+                                                               child);
+        if (maybeShm.isNothing()) {
+          return NS_ERROR_FAILURE;
+        }
+
+        auto surfaceData = maybeShm.value();
+
         // Save the surface data to shared memory.
         if (!surfaceData.IsReadable() || !surfaceData.get<char>()) {
           NS_WARNING("Failed to create shared memory for drag session.");
           return NS_ERROR_FAILURE;
         }
 
         mozilla::Unused <<
           child->SendInvokeDragSession(dataTransfers, aActionType, surfaceData,