371135, oom crashers with big images, r=stuart
authorvladimir@pobox.com
Thu, 10 May 2007 12:58:09 -0700
changeset 1349 a54ae3f1470b868041eb67d4d42003dd44aecb0c
parent 1348 1fbbf6d6afe15c20697cc278a72649999239729a
child 1350 3f7ef7c3a71390c40138d3e7c7b89d9c1fe6871f
push idunknown
push userunknown
push dateunknown
reviewersstuart
bugs371135
milestone1.9a5pre
371135, oom crashers with big images, r=stuart
gfx/src/thebes/nsThebesImage.cpp
gfx/thebes/public/gfxASurface.h
gfx/thebes/src/gfxASurface.cpp
gfx/thebes/src/gfxImageSurface.cpp
gfx/thebes/src/gfxQuartzSurface.cpp
gfx/thebes/src/gfxWindowsSurface.cpp
gfx/thebes/src/gfxXlibSurface.cpp
--- a/gfx/src/thebes/nsThebesImage.cpp
+++ b/gfx/src/thebes/nsThebesImage.cpp
@@ -95,26 +95,36 @@ nsThebesImage::Init(PRInt32 aWidth, PRIn
             break;
     }
 
     mFormat = format;
 
 #ifdef XP_WIN
     if (!ShouldUseImageSurfaces()) {
         mWinSurface = new gfxWindowsSurface(gfxIntSize(mWidth, mHeight), format);
-        mImageSurface = mWinSurface->GetImageSurface();
+        if (mWinSurface && mWinSurface->Status() == 0) {
+            // no error
+            mImageSurface = mWinSurface->GetImageSurface();
+        }
     }
 
     if (!mImageSurface) {
         mWinSurface = nsnull;
         mImageSurface = new gfxImageSurface(gfxIntSize(mWidth, mHeight), format);
     }
 #else
     mImageSurface = new gfxImageSurface(gfxIntSize(mWidth, mHeight), format);
 #endif
+
+    if (!mImageSurface || mImageSurface->Status()) {
+        mImageSurface = nsnull;
+        // guess
+        return NS_ERROR_OUT_OF_MEMORY;
+    }
+
     mStride = mImageSurface->Stride();
 
     return NS_OK;
 }
 
 nsThebesImage::~nsThebesImage()
 {
 }
@@ -281,17 +291,17 @@ NS_IMETHODIMP
 nsThebesImage::LockImagePixels(PRBool aMaskPixels)
 {
     if (aMaskPixels)
         return NS_ERROR_NOT_IMPLEMENTED;
     if ((mOptSurface || mSinglePixel) && !mImageSurface) {
         // Recover the pixels
         mImageSurface = new gfxImageSurface(gfxIntSize(mWidth, mHeight),
                                             gfxImageSurface::ImageFormatARGB32);
-        if (!mImageSurface)
+        if (!mImageSurface || mImageSurface->Status())
             return NS_ERROR_OUT_OF_MEMORY;
         nsRefPtr<gfxContext> context = new gfxContext(mImageSurface);
         if (!context) {
             mImageSurface = nsnull;
             return NS_ERROR_OUT_OF_MEMORY;
         }
         context->SetOperator(gfxContext::OPERATOR_SOURCE);
         if (mSinglePixel)
@@ -459,16 +469,21 @@ nsThebesImage::ThebesDrawTile(gfxContext
             height = mHeight + yPadding;
 
             // Reject over-wide or over-tall images.
             if (!AllowedImageSize(width, height))
                 return NS_ERROR_FAILURE;
 
             surface = new gfxImageSurface(gfxIntSize(width, height),
                                           gfxASurface::ImageFormatARGB32);
+            if (!surface || surface->Status()) {
+                thebesContext->SetMatrix(savedCTM);
+                return NS_ERROR_OUT_OF_MEMORY;
+            }
+
             tmpSurfaceGrip = surface;
 
             nsRefPtr<gfxContext> tmpContext = new gfxContext(surface);
             if (mSinglePixel) {
                 tmpContext->SetColor(mSinglePixelColor);
             } else {
                 tmpContext->SetSource(ThebesSurface());
             }
--- a/gfx/thebes/public/gfxASurface.h
+++ b/gfx/thebes/public/gfxASurface.h
@@ -89,17 +89,20 @@ public:
         CONTENT_ALPHA       = 0x2000,
         CONTENT_COLOR_ALPHA = 0x3000
     } gfxContentType;
 
     /* Wrap the given cairo surface and return a gfxASurface for it */
     static already_AddRefed<gfxASurface> Wrap(cairo_surface_t *csurf);
 
     /*** this DOES NOT addref the surface */
-    cairo_surface_t *CairoSurface() { return mSurface; }
+    cairo_surface_t *CairoSurface() {
+        NS_ASSERTION(mSurface != nsnull, "gfxASurface::CairoSurface called with mSurface == nsnull!");
+        return mSurface;
+    }
 
     gfxSurfaceType GetType() const;
 
     gfxContentType GetContentType() const;
 
     void SetDeviceOffset(const gfxPoint& offset);
     gfxPoint GetDeviceOffset() const;
 
@@ -116,29 +119,42 @@ public:
 
     void SetData(const cairo_user_data_key_t *key,
                  void *user_data,
                  thebes_destroy_func_t destroy);
     void *GetData(const cairo_user_data_key_t *key);
 
     virtual void Finish();
 
+    int Status();
+
+    /* Make sure that the given dimensions don't overflow a 32-bit signed int
+     * using 4 bytes per pixel; optionally, make sure that either dimension
+     * doesn't exceed the given limit.
+     */
+    static PRBool CheckSurfaceSize(const gfxIntSize& sz, PRInt32 limit = 0);
+
 protected:
+    gfxASurface() : mSurface(nsnull), mFloatingRefs(0), mSurfaceValid(PR_FALSE) { }
+
     static gfxASurface* GetSurfaceWrapper(cairo_surface_t *csurf);
     static void SetSurfaceWrapper(cairo_surface_t *csurf, gfxASurface *asurf);
 
     void Init(cairo_surface_t *surface, PRBool existingSurface = PR_FALSE);
 
     virtual ~gfxASurface() {
     }
 private:
+    static void SurfaceDestroyFunc(void *data);
+
     cairo_surface_t *mSurface;
-    PRPackedBool mHasFloatingRef;
+    PRInt32 mFloatingRefs;
 
-    static void SurfaceDestroyFunc(void *data);
+protected:
+    PRPackedBool mSurfaceValid;
 };
 
 /**
  * An Unknown surface; used to wrap unknown cairo_surface_t returns from cairo
  */
 class THEBES_API gfxUnknownSurface : public gfxASurface {
 public:
     gfxUnknownSurface(cairo_surface_t *surf) {
--- a/gfx/thebes/src/gfxASurface.cpp
+++ b/gfx/thebes/src/gfxASurface.cpp
@@ -60,40 +60,57 @@ static cairo_user_data_key_t gfxasurface
 
 // Surfaces use refcounting that's tied to the cairo surface refcnt, to avoid
 // refcount mismatch issues.
 nsrefcnt
 gfxASurface::AddRef(void)
 {
     NS_PRECONDITION(mSurface != nsnull, "gfxASurface::AddRef without mSurface");
 
-    if (mHasFloatingRef) {
-        // eat the floating ref
-        mHasFloatingRef = PR_FALSE;
+    if (mSurfaceValid) {
+        if (mFloatingRefs) {
+            // eat a floating ref
+            mFloatingRefs--;
+        } else {
+            cairo_surface_reference(mSurface);
+        }
+
+        return (nsrefcnt) cairo_surface_get_reference_count(mSurface);
     } else {
-        cairo_surface_reference(mSurface);
+        // the surface isn't valid, but we still need to refcount
+        // the gfxASurface
+        return ++mFloatingRefs;
     }
-
-    return (nsrefcnt) cairo_surface_get_reference_count(mSurface);
 }
 
 nsrefcnt
 gfxASurface::Release(void)
 {
-    NS_PRECONDITION(!mHasFloatingRef, "gfxASurface::Release while floating ref still outstanding!");
     NS_PRECONDITION(mSurface != nsnull, "gfxASurface::Release without mSurface");
-    // Note that there is a destructor set on user data for mSurface,
-    // which will delete this gfxASurface wrapper when the surface's refcount goes
-    // out of scope.
-    nsrefcnt refcnt = (nsrefcnt) cairo_surface_get_reference_count(mSurface);
-    cairo_surface_destroy(mSurface);
+
+    if (mSurfaceValid) {
+        NS_ASSERTION(mFloatingRefs == 0, "gfxASurface::Release with floating refs still hanging around!");
+
+        // Note that there is a destructor set on user data for mSurface,
+        // which will delete this gfxASurface wrapper when the surface's refcount goes
+        // out of scope.
+        nsrefcnt refcnt = (nsrefcnt) cairo_surface_get_reference_count(mSurface);
+        cairo_surface_destroy(mSurface);
 
-    // |this| may not be valid any more, don't use it!
+        // |this| may not be valid any more, don't use it!
 
-    return --refcnt;
+        return --refcnt;
+    } else {
+        if (--mFloatingRefs == 0) {
+            delete this;
+            return 0;
+        }
+
+        return mFloatingRefs;
+    }
 }
 
 void
 gfxASurface::SurfaceDestroyFunc(void *data) {
     gfxASurface *surf = (gfxASurface*) data;
     // fprintf (stderr, "Deleting wrapper for %p (wrapper: %p)\n", surf->mSurface, data);
     delete surf;
 }
@@ -152,24 +169,32 @@ gfxASurface::Wrap (cairo_surface_t *csur
 
     NS_ADDREF(result);
     return result;
 }
 
 void
 gfxASurface::Init(cairo_surface_t* surface, PRBool existingSurface)
 {
+    if (cairo_surface_status(surface)) {
+        // the surface has an error on it
+        mSurfaceValid = PR_FALSE;
+        cairo_surface_destroy(surface);
+        return;
+    }
+
     SetSurfaceWrapper(surface, this);
 
     mSurface = surface;
+    mSurfaceValid = PR_TRUE;
 
     if (existingSurface) {
-        mHasFloatingRef = PR_FALSE;
+        mFloatingRefs = 0;
     } else {
-        mHasFloatingRef = PR_TRUE;
+        mFloatingRefs = 1;
     }
 }
 
 gfxASurface::gfxSurfaceType
 gfxASurface::GetType() const
 {
     return (gfxSurfaceType)cairo_surface_get_type(mSurface);
 }
@@ -210,17 +235,16 @@ gfxASurface::MarkDirty()
 void
 gfxASurface::MarkDirty(const gfxRect& r)
 {
     cairo_surface_mark_dirty_rectangle(mSurface,
                                        (int) r.pos.x, (int) r.pos.y,
                                        (int) r.size.width, (int) r.size.height);
 }
 
-
 void
 gfxASurface::SetData(const cairo_user_data_key_t *key,
                      void *user_data,
                      thebes_destroy_func_t destroy)
 {
     cairo_surface_set_user_data(mSurface, key, user_data, destroy);
 }
 
@@ -230,8 +254,42 @@ gfxASurface::GetData(const cairo_user_da
     return cairo_surface_get_user_data(mSurface, key);
 }
 
 void
 gfxASurface::Finish()
 {
     cairo_surface_finish(mSurface);
 }
+
+int
+gfxASurface::Status()
+{
+    if (!mSurfaceValid)
+        return -1;
+
+    return cairo_surface_status(mSurface);
+}
+
+/* static */
+PRBool
+gfxASurface::CheckSurfaceSize(const gfxIntSize& sz, PRInt32 limit)
+{
+    if (sz.width <= 0 || sz.height <= 0)
+        return PR_FALSE;
+
+    // check to make sure we don't overflow a 32-bit
+    PRInt32 tmp = sz.width * sz.height;
+    if (tmp / sz.height != sz.width)
+        return PR_FALSE;
+
+    // always assume 4-byte stride
+    tmp = tmp * 4;
+    if (tmp / 4 != sz.width * sz.height)
+        return PR_FALSE;
+
+    // reject images with sides bigger than limit
+    if (limit &&
+        (sz.width > limit || sz.height > limit))
+        return PR_FALSE;
+
+    return PR_TRUE;
+}
--- a/gfx/thebes/src/gfxImageSurface.cpp
+++ b/gfx/thebes/src/gfxImageSurface.cpp
@@ -30,35 +30,42 @@
  * use your version of this file under the terms of the MPL, indicate your
  * decision by deleting the provisions above and replace them with the notice
  * and other provisions required by the GPL or the LGPL. If you do not delete
  * the provisions above, a recipient may use your version of this file under
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
 
+#include "prmem.h"
+
 #include "gfxImageSurface.h"
 
 #include "cairo.h"
 
 gfxImageSurface::gfxImageSurface(const gfxIntSize& size, gfxImageFormat format) :
     mSize(size), mFormat(format)
 {
-    long stride = ComputeStride();
-    mData = new unsigned char[mSize.height * stride];
+    mStride = ComputeStride();
+
+    if (!CheckSurfaceSize(size))
+        return;
+
+    mData = (unsigned char *) malloc(mSize.height * mStride);
+    if (!mData)
+        return;
+
     mOwnsData = PR_TRUE;
 
     cairo_surface_t *surface =
         cairo_image_surface_create_for_data((unsigned char*)mData,
                                             (cairo_format_t)format,
                                             mSize.width,
                                             mSize.height,
-                                            stride);
-    mStride = stride;
-
+                                            mStride);
     Init(surface);
 }
 
 gfxImageSurface::gfxImageSurface(cairo_surface_t *csurf)
 {
     mSize.width = cairo_image_surface_get_width(csurf);
     mSize.height = cairo_image_surface_get_height(csurf);
     mData = cairo_image_surface_get_data(csurf);
@@ -66,18 +73,23 @@ gfxImageSurface::gfxImageSurface(cairo_s
     mOwnsData = PR_FALSE;
     mStride = cairo_image_surface_get_stride(csurf);
 
     Init(csurf, PR_TRUE);
 }
 
 gfxImageSurface::~gfxImageSurface()
 {
-    if (mOwnsData)
-        delete[] mData;
+    if (!mSurfaceValid)
+        return;
+
+    if (mOwnsData) {
+        free(mData);
+        mData = nsnull;
+    }
 }
 
 long
 gfxImageSurface::ComputeStride() const
 {
     long stride;
 
     if (mFormat == ImageFormatARGB32)
--- a/gfx/thebes/src/gfxQuartzSurface.cpp
+++ b/gfx/thebes/src/gfxQuartzSurface.cpp
@@ -37,16 +37,19 @@
 
 #include "gfxQuartzSurface.h"
 
 #include "cairo-quartz.h"
 
 gfxQuartzSurface::gfxQuartzSurface(const gfxSize& size, gfxImageFormat format)
     : mSize(size)
 {
+    if (!CheckSurfaceSize(size))
+        return;
+
     cairo_surface_t *surf = cairo_quartz_surface_create
         ((cairo_format_t) format, floor(size.width), floor(size.height));
 
     mCGContext = cairo_quartz_surface_get_cg_context (surf);
 
     CGContextRetain(mCGContext);
 
     Init(surf);
--- a/gfx/thebes/src/gfxWindowsSurface.cpp
+++ b/gfx/thebes/src/gfxWindowsSurface.cpp
@@ -55,36 +55,34 @@ gfxWindowsSurface::gfxWindowsSurface(HDC
     mOwnsDC(deleteDC), mDC(dc),mWnd(nsnull)
 {
     Init(cairo_win32_surface_create(mDC));
 }
 
 gfxWindowsSurface::gfxWindowsSurface(const gfxIntSize& size, gfxImageFormat imageFormat) :
     mOwnsDC(PR_FALSE), mWnd(nsnull)
 {
+    if (!CheckSurfaceSize(size))
+        return;
+
     cairo_surface_t *surf = cairo_win32_surface_create_with_dib((cairo_format_t)imageFormat,
                                                                 size.width, size.height);
-    if (!surf || cairo_surface_status(surf)) {
-        fprintf (stderr, "++++++++++++ gfxWindowsSurface: DIB surface creation failed!\n");
-    }
-
     Init(surf);
 
     mDC = cairo_win32_surface_get_dc(CairoSurface());
 }
 
 gfxWindowsSurface::gfxWindowsSurface(HDC dc, const gfxIntSize& size, gfxImageFormat imageFormat) :
     mOwnsDC(PR_FALSE), mWnd(nsnull)
 {
+    if (!CheckSurfaceSize(size))
+        return;
+
     cairo_surface_t *surf = cairo_win32_surface_create_with_ddb(dc, (cairo_format_t)imageFormat,
                                                                 size.width, size.height);
-    if (!surf || cairo_surface_status(surf)) {
-        fprintf (stderr, "++++++++++++ gfxWindowsSurface: DDB surface creation failed!\n");
-    }
-
     Init(surf);
 
     mDC = cairo_win32_surface_get_dc(CairoSurface());
 }
 
 
 gfxWindowsSurface::gfxWindowsSurface(cairo_surface_t *csurf) :
     mOwnsDC(PR_FALSE), mWnd(nsnull)
--- a/gfx/thebes/src/gfxXlibSurface.cpp
+++ b/gfx/thebes/src/gfxXlibSurface.cpp
@@ -46,50 +46,61 @@ static cairo_user_data_key_t pixmap_free
 
 typedef struct {
     Display* dpy;
     Pixmap pixmap;
 } pixmap_free_struct;
 
 static void pixmap_free_func (void *);
 
+#define XLIB_IMAGE_SIDE_SIZE_LIMIT 0xffff
+
 gfxXlibSurface::gfxXlibSurface(Display *dpy, Drawable drawable, Visual *visual)
     : mPixmapTaken(PR_FALSE), mDisplay(dpy), mDrawable(drawable)
 {
     DoSizeQuery();
     cairo_surface_t *surf = cairo_xlib_surface_create(dpy, drawable, visual, mSize.width, mSize.height);
     Init(surf);
 }
 
 gfxXlibSurface::gfxXlibSurface(Display *dpy, Drawable drawable, Visual *visual, const gfxIntSize& size)
     : mPixmapTaken(PR_FALSE), mDisplay(dpy), mDrawable(drawable), mSize(size)
 {
+    if (!CheckSurfaceSize(size, XLIB_IMAGE_SIDE_SIZE_LIMIT))
+        return;
+
     cairo_surface_t *surf = cairo_xlib_surface_create(dpy, drawable, visual, mSize.width, mSize.height);
     Init(surf);
 }
 
 gfxXlibSurface::gfxXlibSurface(Display *dpy, Visual *visual, const gfxIntSize& size)
     : mPixmapTaken(PR_FALSE), mDisplay(dpy), mSize(size)
 
 {
+    if (!CheckSurfaceSize(size, XLIB_IMAGE_SIDE_SIZE_LIMIT))
+        return;
+
     mDrawable = (Drawable)XCreatePixmap(dpy,
                                         RootWindow(dpy, DefaultScreen(dpy)),
                                         mSize.width, mSize.height,
                                         DefaultDepth(dpy, DefaultScreen(dpy)));
 
     cairo_surface_t *surf = cairo_xlib_surface_create(dpy, mDrawable, visual, mSize.width, mSize.height);
 
     Init(surf);
     TakePixmap();
 }
 
 gfxXlibSurface::gfxXlibSurface(Display *dpy, Drawable drawable, XRenderPictFormat *format,
                                const gfxIntSize& size)
     : mPixmapTaken(PR_FALSE), mDisplay(dpy), mDrawable(drawable), mSize(size)
 {
+    if (!CheckSurfaceSize(size, XLIB_IMAGE_SIDE_SIZE_LIMIT))
+        return;
+
     cairo_surface_t *surf = cairo_xlib_surface_create_with_xrender_format(dpy, drawable,
                                                                           ScreenOfDisplay(dpy,DefaultScreen(dpy)),
                                                                           format, mSize.width, mSize.height);
     Init(surf);
 }
 
 gfxXlibSurface::gfxXlibSurface(Display *dpy, XRenderPictFormat *format, const gfxIntSize& size)
     : mPixmapTaken(PR_FALSE), mDisplay(dpy), mSize(size)