Bug 1067998 - Fix OOM crash in gfxAlphaBoxBlur::Init on large blur surface. r=Bas, a=sylvestre
authorJonathan Watt <jwatt@jwatt.org>
Wed, 17 Sep 2014 08:46:42 +0100
changeset 216774 023a362fab21
parent 216773 f44f06112715
child 216775 9dd2e1834651
push id3909
push userjwatt@jwatt.org
push date2014-09-18 10:33 +0000
treeherdermozilla-beta@023a362fab21 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersBas, sylvestre
bugs1067998
milestone33.0
Bug 1067998 - Fix OOM crash in gfxAlphaBoxBlur::Init on large blur surface. r=Bas, a=sylvestre
gfx/2d/Blur.cpp
gfx/2d/Blur.h
gfx/thebes/gfxBlur.cpp
--- a/gfx/2d/Blur.cpp
+++ b/gfx/2d/Blur.cpp
@@ -330,17 +330,17 @@ AlphaBoxBlur::RoundUpToMultipleOf4(int32
 
 AlphaBoxBlur::AlphaBoxBlur(const Rect& aRect,
                            const IntSize& aSpreadRadius,
                            const IntSize& aBlurRadius,
                            const Rect* aDirtyRect,
                            const Rect* aSkipRect)
  : mSpreadRadius(aSpreadRadius),
    mBlurRadius(aBlurRadius),
-   mSurfaceAllocationSize(-1)
+   mSurfaceAllocationSize(0)
 {
   Rect rect(aRect);
   rect.Inflate(Size(aBlurRadius + aSpreadRadius));
   rect.RoundOut();
 
   if (aDirtyRect) {
     // If we get passed a dirty rect from layout, we can minimize the
     // shadow size and make painting faster.
@@ -395,17 +395,17 @@ AlphaBoxBlur::AlphaBoxBlur(const Rect& a
                            int32_t aStride,
                            float aSigmaX,
                            float aSigmaY)
   : mRect(int32_t(aRect.x), int32_t(aRect.y),
           int32_t(aRect.width), int32_t(aRect.height)),
     mSpreadRadius(),
     mBlurRadius(CalculateBlurRadius(Point(aSigmaX, aSigmaY))),
     mStride(aStride),
-    mSurfaceAllocationSize(-1)
+    mSurfaceAllocationSize(0)
 {
   IntRect intRect;
   if (aRect.ToIntRect(&intRect)) {
     size_t minDataSize = BufferSizeFromStrideAndHeight(intRect.width, intRect.height);
     if (minDataSize != 0) {
       mSurfaceAllocationSize = minDataSize;
     }
   }
@@ -440,17 +440,17 @@ AlphaBoxBlur::GetDirtyRect()
 {
   if (mHasDirtyRect) {
     return &mDirtyRect;
   }
 
   return nullptr;
 }
 
-int32_t
+size_t
 AlphaBoxBlur::GetSurfaceAllocationSize() const
 {
   return mSurfaceAllocationSize;
 }
 
 void
 AlphaBoxBlur::Blur(uint8_t* aData)
 {
--- a/gfx/2d/Blur.h
+++ b/gfx/2d/Blur.h
@@ -89,22 +89,22 @@ public:
   /**
    * Return a pointer to a dirty rect, as passed in to the constructor, or nullptr
    * if none was passed in.
    */
   Rect* GetDirtyRect();
 
   /**
    * Return the minimum buffer size that should be given to Blur() method.  If
-   * negative, the class is not properly setup for blurring.  Note that this
+   * zero, the class is not properly setup for blurring.  Note that this
    * includes the extra three bytes on top of the stride*width, where something
    * like gfxImageSurface::GetDataSize() would report without it, even if it 
    * happens to have the extra bytes.
    */
-  int32_t GetSurfaceAllocationSize() const;
+  size_t GetSurfaceAllocationSize() const;
 
   /**
    * Perform the blur in-place on the surface backed by specified 8-bit
    * alpha surface data. The size must be at least that returned by
    * GetSurfaceAllocationSize() or bad things will happen.
    */
   void Blur(uint8_t* aData);
 
@@ -157,17 +157,17 @@ private:
   /**
    * The stride of the data passed to Blur()
    */
   int32_t mStride;
 
   /**
    * The minimum size of the buffer needed for the Blur() operation.
    */
-  int32_t mSurfaceAllocationSize;
+  size_t mSurfaceAllocationSize;
 
   /**
    * Whether mDirtyRect contains valid data.
    */
   bool mHasDirtyRect;
 };
 
 }
--- a/gfx/thebes/gfxBlur.cpp
+++ b/gfx/thebes/gfxBlur.cpp
@@ -48,25 +48,28 @@ gfxAlphaBoxBlur::Init(const gfxRect& aRe
     if (aSkipRect) {
       skipRect = new mozilla::gfx::Rect(Float(aSkipRect->x),
                                         Float(aSkipRect->y),
                                         Float(aSkipRect->width),
                                         Float(aSkipRect->height));
     }
 
     mBlur = new AlphaBoxBlur(rect, spreadRadius, blurRadius, dirtyRect, skipRect);
-    int32_t blurDataSize = mBlur->GetSurfaceAllocationSize();
-    if (blurDataSize <= 0)
+    size_t blurDataSize = mBlur->GetSurfaceAllocationSize();
+    if (blurDataSize == 0)
         return nullptr;
 
     IntSize size = mBlur->GetSize();
 
     // Make an alpha-only surface to draw on. We will play with the data after
     // everything is drawn to create a blur effect.
-    mData = new unsigned char[blurDataSize];
+    mData = new (std::nothrow) unsigned char[blurDataSize];
+    if (!mData) {
+        return nullptr;
+    }
     memset(mData, 0, blurDataSize);
 
     mozilla::RefPtr<DrawTarget> dt =
         gfxPlatform::GetPlatform()->CreateDrawTargetForData(mData, size,
                                                             mBlur->GetStride(),
                                                             SurfaceFormat::A8);
     if (!dt) {
         return nullptr;