Bug 1224254 - Don't try to allocate unreasonably large textures. r=Bas, a=lizzard
authorNicolas Silva <nsilva@mozilla.com>
Wed, 18 Nov 2015 16:59:11 +0100
changeset 305581 c642a03ec545403a18a840001295c6708829691c
parent 305580 7d63a6f2d3592aafe729a6bc684935d5f80ace5e
child 305582 722c66ce28ff0da90c0df24e79215f6687498741
push id1001
push userraliiev@mozilla.com
push dateMon, 18 Jan 2016 19:06:03 +0000
treeherdermozilla-release@8b89261f3ac4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersBas, lizzard
bugs1224254
milestone44.0a2
Bug 1224254 - Don't try to allocate unreasonably large textures. r=Bas, a=lizzard
gfx/2d/2D.h
gfx/2d/Factory.cpp
gfx/layers/ImageDataSerializer.cpp
gfx/layers/YCbCrImageDataSerializer.cpp
gfx/layers/client/TextureClient.cpp
gfx/thebes/gfxPlatform.cpp
gfx/thebes/gfxPrefs.h
--- a/gfx/2d/2D.h
+++ b/gfx/2d/2D.h
@@ -1123,32 +1123,51 @@ struct Tile
 };
 
 struct TileSet
 {
   Tile* mTiles;
   size_t mTileCount;
 };
 
+struct Config {
+  LogForwarder* mLogForwarder;
+  int32_t mMaxTextureSize;
+  int32_t mMaxAllocSize;
+
+  Config()
+  : mLogForwarder(nullptr)
+  , mMaxTextureSize(8192)
+  , mMaxAllocSize(52000000)
+  {}
+};
+
 class GFX2D_API Factory
 {
 public:
+  static void Init(const Config& aConfig);
+  static void ShutDown();
+
   static bool HasSSE2();
 
   /** Make sure that the given dimensions don't overflow a 32-bit signed int
    * using 4 bytes per pixel; optionally, make sure that either dimension
    * doesn't exceed the given limit.
    */
-  static bool CheckSurfaceSize(const IntSize &sz, int32_t limit = 0);
+  static bool CheckSurfaceSize(const IntSize &sz,
+                               int32_t limit = 0,
+                               int32_t allocLimit = 0);
 
   /** Make sure the given dimension satisfies the CheckSurfaceSize and is
    * within 8k limit.  The 8k value is chosen a bit randomly.
    */
   static bool ReasonableSurfaceSize(const IntSize &aSize);
 
+  static bool AllowedSurfaceSize(const IntSize &aSize);
+
   static already_AddRefed<DrawTarget> CreateDrawTargetForCairoSurface(cairo_surface_t* aSurface, const IntSize& aSize, SurfaceFormat* aFormat = nullptr);
 
   static already_AddRefed<DrawTarget>
     CreateDrawTarget(BackendType aBackend, const IntSize &aSize, SurfaceFormat aFormat);
 
   static already_AddRefed<DrawTarget>
     CreateRecordingDrawTarget(DrawEventRecorder *aRecorder, DrawTarget *aDT);
      
@@ -1212,20 +1231,20 @@ public:
 
   static void SetGlobalEventRecorder(DrawEventRecorder *aRecorder);
 
   // This is a little hacky at the moment, but we want to have this data. Bug 1068613.
   static void SetLogForwarder(LogForwarder* aLogFwd);
 
   static uint32_t GetMaxSurfaceSize(BackendType aType);
 
-  static LogForwarder* GetLogForwarder() { return mLogForwarder; }
+  static LogForwarder* GetLogForwarder() { return sConfig->mLogForwarder; }
 
 private:
-  static LogForwarder* mLogForwarder;
+  static Config* sConfig;
 public:
 
 #ifdef USE_SKIA_GPU
   static already_AddRefed<DrawTarget>
     CreateDrawTargetSkiaWithGrContext(GrContext* aGrContext,
                                       const IntSize &aSize,
                                       SurfaceFormat aFormat);
 #endif
--- a/gfx/2d/Factory.cpp
+++ b/gfx/2d/Factory.cpp
@@ -183,16 +183,45 @@ void PreferenceAccess::SetAccess(Prefere
 #ifdef WIN32
 ID3D10Device1 *Factory::mD3D10Device;
 ID3D11Device *Factory::mD3D11Device;
 ID2D1Device *Factory::mD2D1Device;
 #endif
 
 DrawEventRecorder *Factory::mRecorder;
 
+mozilla::gfx::Config* Factory::sConfig = nullptr;
+
+void
+Factory::Init(const Config& aConfig)
+{
+  MOZ_ASSERT(!sConfig);
+  sConfig = new Config(aConfig);
+
+  // Make sure we don't completely break rendering because of a typo in the
+  // pref or whatnot.
+  const int32_t kMinAllocPref = 10000000;
+  const int32_t kMinSizePref = 2048;
+  if (sConfig->mMaxAllocSize < kMinAllocPref) {
+    sConfig->mMaxAllocSize = kMinAllocPref;
+  }
+  if (sConfig->mMaxTextureSize < kMinSizePref) {
+    sConfig->mMaxTextureSize = kMinSizePref;
+  }
+}
+
+void
+Factory::ShutDown()
+{
+  if (sConfig) {
+    delete sConfig;
+    sConfig = nullptr;
+  }
+}
+
 bool
 Factory::HasSSE2()
 {
 #if defined(__SSE2__) || defined(_M_X64) || \
     (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
   // gcc with -msse2 (default on OSX and x86-64)
   // cl.exe with -arch:SSE2 (default on x64 compiler)
   return true;
@@ -217,30 +246,44 @@ Factory::HasSSE2()
 inline int LoggerOptionsBasedOnSize(const IntSize& aSize)
 {
   return CriticalLog::DefaultOptions(Factory::ReasonableSurfaceSize(aSize));
 }
 
 bool
 Factory::ReasonableSurfaceSize(const IntSize &aSize)
 {
-  return Factory::CheckSurfaceSize(aSize,8192);
+  return Factory::CheckSurfaceSize(aSize, 8192);
 }
 
 bool
-Factory::CheckSurfaceSize(const IntSize &sz, int32_t limit)
+Factory::AllowedSurfaceSize(const IntSize &aSize)
+{
+  if (sConfig) {
+    return Factory::CheckSurfaceSize(aSize,
+                                     sConfig->mMaxTextureSize,
+                                     sConfig->mMaxAllocSize);
+  }
+
+  return CheckSurfaceSize(aSize);
+}
+
+bool
+Factory::CheckSurfaceSize(const IntSize &sz,
+                          int32_t extentLimit,
+                          int32_t allocLimit)
 {
   if (sz.width <= 0 || sz.height <= 0) {
     gfxDebug() << "Surface width or height <= 0!";
     return false;
   }
 
   // reject images with sides bigger than limit
-  if (limit && (sz.width > limit || sz.height > limit)) {
-    gfxDebug() << "Surface size too large (exceeds caller's limit)!";
+  if (extentLimit && (sz.width > extentLimit || sz.height > extentLimit)) {
+    gfxDebug() << "Surface size too large (exceeds extent limit)!";
     return false;
   }
 
   // make sure the surface area doesn't overflow a int32_t
   CheckedInt<int32_t> tmp = sz.width;
   tmp *= sz.height;
   if (!tmp.isValid()) {
     gfxDebug() << "Surface size too large (would overflow)!";
@@ -262,23 +305,28 @@ Factory::CheckSurfaceSize(const IntSize 
 
   CheckedInt<int32_t> numBytes = GetAlignedStride<16>(sz.width * 4);
   numBytes *= sz.height;
   if (!numBytes.isValid()) {
     gfxDebug() << "Surface size too large (allocation size would overflow int32_t)!";
     return false;
   }
 
+  if (allocLimit && allocLimit < numBytes.value()) {
+    gfxDebug() << "Surface size too large (exceeds allocation limit)!";
+    return false;
+  }
+
   return true;
 }
 
 already_AddRefed<DrawTarget>
 Factory::CreateDrawTarget(BackendType aBackend, const IntSize &aSize, SurfaceFormat aFormat)
 {
-  if (!CheckSurfaceSize(aSize)) {
+  if (!AllowedSurfaceSize(aSize)) {
     gfxCriticalError(LoggerOptionsBasedOnSize(aSize)) << "Failed to allocate a surface due to invalid size " << aSize;
     return nullptr;
   }
 
   RefPtr<DrawTarget> retVal;
   switch (aBackend) {
 #ifdef WIN32
   case BackendType::DIRECT2D:
@@ -359,17 +407,17 @@ Factory::CreateRecordingDrawTarget(DrawE
 already_AddRefed<DrawTarget>
 Factory::CreateDrawTargetForData(BackendType aBackend,
                                  unsigned char *aData,
                                  const IntSize &aSize,
                                  int32_t aStride,
                                  SurfaceFormat aFormat)
 {
   MOZ_ASSERT(aData);
-  if (!CheckSurfaceSize(aSize)) {
+  if (!AllowedSurfaceSize(aSize)) {
     gfxCriticalError(LoggerOptionsBasedOnSize(aSize)) << "Failed to allocate a surface due to invalid size " << aSize;
     return nullptr;
   }
 
   RefPtr<DrawTarget> retVal;
 
   switch (aBackend) {
 #ifdef USE_SKIA
@@ -835,17 +883,17 @@ Factory::CreateWrappingDataSourceSurface
   return nullptr;
 }
 
 already_AddRefed<DataSourceSurface>
 Factory::CreateDataSourceSurface(const IntSize &aSize,
                                  SurfaceFormat aFormat,
                                  bool aZero)
 {
-  if (!CheckSurfaceSize(aSize)) {
+  if (!AllowedSurfaceSize(aSize)) {
     gfxCriticalError(LoggerOptionsBasedOnSize(aSize)) << "Failed to allocate a surface due to invalid size " << aSize;
     return nullptr;
   }
 
   RefPtr<SourceSurfaceAlignedRawData> newSurf = new SourceSurfaceAlignedRawData();
   if (newSurf->Init(aSize, aFormat, aZero)) {
     return newSurf.forget();
   }
@@ -881,24 +929,23 @@ Factory::CreateEventRecorderForFile(cons
 }
 
 void
 Factory::SetGlobalEventRecorder(DrawEventRecorder *aRecorder)
 {
   mRecorder = aRecorder;
 }
 
-LogForwarder* Factory::mLogForwarder = nullptr;
-
 // static
 void
 Factory::SetLogForwarder(LogForwarder* aLogFwd) {
-  mLogForwarder = aLogFwd;
+  sConfig->mLogForwarder = aLogFwd;
 }
 
+
 // static
 void
 CriticalLogger::OutputMessage(const std::string &aString,
                               int aLevel, bool aNoNewline)
 {
   if (Factory::GetLogForwarder()) {
     Factory::GetLogForwarder()->Log(aString);
   }
--- a/gfx/layers/ImageDataSerializer.cpp
+++ b/gfx/layers/ImageDataSerializer.cpp
@@ -79,47 +79,50 @@ ComputeStride(SurfaceFormat aFormat, int
   return GetAlignedStride<4>(size.value());
 }
 
 uint32_t
 ImageDataSerializerBase::ComputeMinBufferSize(IntSize aSize,
                                               SurfaceFormat aFormat)
 {
   MOZ_ASSERT(aSize.height >= 0 && aSize.width >= 0);
-  if (aSize.height <= 0 || aSize.width <= 0) {
-    gfxDebug() << "Non-positive image buffer size request " << aSize.width << "x" << aSize.height;
+
+  // This takes care of checking whether there could be overflow
+  // with enough margin for the metadata.
+  if (!gfx::Factory::AllowedSurfaceSize(aSize)) {
     return 0;
   }
 
-  CheckedInt<int32_t> bufsize = ComputeStride(aFormat, aSize.width);
-  bufsize *= aSize.height;
+  int32_t bufsize = GetAlignedStride<16>(ComputeStride(aFormat, aSize.width)
+                                         * aSize.height)
+                  + SurfaceBufferInfo::GetOffset();
 
-  if (!bufsize.isValid() || bufsize.value() <= 0) {
-    gfxDebug() << "Buffer size overflow " << aSize.width << "x" << aSize.height;
+  if (bufsize < 0) {
+    // This should not be possible thanks to Factory::AllowedSurfaceSize
     return 0;
   }
 
-  return SurfaceBufferInfo::GetOffset()
-       + GetAlignedStride<16>(bufsize.value());
+  return bufsize;
 }
 
 void
 ImageDataSerializerBase::Validate()
 {
   mIsValid = false;
   if (!mData) {
     return;
   }
   SurfaceBufferInfo* info = GetBufferInfo(mData, mDataSize);
   if (!info) {
     return;
   }
   size_t requiredSize =
            ComputeMinBufferSize(IntSize(info->width, info->height), info->format);
-  mIsValid = requiredSize <= mDataSize;
+
+  mIsValid = !!requiredSize && requiredSize <= mDataSize;
 }
 
 uint8_t*
 ImageDataSerializerBase::GetData()
 {
   MOZ_ASSERT(IsValid());
   return mData + SurfaceBufferInfo::GetOffset();
 }
--- a/gfx/layers/YCbCrImageDataSerializer.cpp
+++ b/gfx/layers/YCbCrImageDataSerializer.cpp
@@ -146,16 +146,23 @@ YCbCrImageDataDeserializerBase::ComputeM
                                                      const gfx::IntSize& aCbCrSize,
                                                      uint32_t aCbCrStride)
 {
   MOZ_ASSERT(aYSize.height >= 0 && aYSize.width >= 0);
   if (aYSize.height <= 0 || aYSize.width <= 0 || aCbCrSize.height <= 0 || aCbCrSize.width <= 0) {
     gfxDebug() << "Non-positive YCbCr buffer size request " << aYSize.height << "x" << aYSize.width << ", " << aCbCrSize.height << "x" << aCbCrSize.width;
     return 0;
   }
+
+  if (!gfx::Factory::AllowedSurfaceSize(aYSize) ||
+      aCbCrSize.width > aYSize.width ||
+      aCbCrSize.height > aYSize.height) {
+    return 0;
+  }
+
   return ComputeOffset(aYSize.height, aYStride)
          + 2 * ComputeOffset(aCbCrSize.height, aCbCrStride)
          + MOZ_ALIGN_WORD(sizeof(YCbCrBufferInfo));
 }
 
 // Minimum required shmem size in bytes
 size_t
 YCbCrImageDataDeserializerBase::ComputeMinBufferSize(const gfx::IntSize& aYSize,
--- a/gfx/layers/client/TextureClient.cpp
+++ b/gfx/layers/client/TextureClient.cpp
@@ -381,16 +381,20 @@ TextureClient::CreateForDrawing(Composit
                                 TextureFlags aTextureFlags,
                                 TextureAllocationFlags aAllocFlags)
 {
   MOZ_ASSERT(aAllocator->IPCOpen());
   if (!aAllocator || !aAllocator->IPCOpen()) {
     return nullptr;
   }
 
+  if (!gfx::Factory::AllowedSurfaceSize(aSize)) {
+    return nullptr;
+  }
+
   LayersBackend parentBackend = aAllocator->GetCompositorBackendType();
   gfx::BackendType moz2DBackend = BackendTypeForBackendSelector(parentBackend, aSelector);
 
   RefPtr<TextureClient> texture;
 
 #if defined(MOZ_WIDGET_GONK) || defined(XP_WIN)
   int32_t maxTextureSize = aAllocator->GetMaxTextureSize();
 #endif
@@ -492,16 +496,20 @@ TextureClient::CreateForRawBufferAccess(
                                         TextureFlags aTextureFlags,
                                         TextureAllocationFlags aAllocFlags)
 {
   MOZ_ASSERT(aAllocator->IPCOpen());
   if (!aAllocator || !aAllocator->IPCOpen()) {
     return nullptr;
   }
 
+  if (!gfx::Factory::AllowedSurfaceSize(aSize)) {
+    return nullptr;
+  }
+
   RefPtr<BufferTextureClient> texture =
     CreateBufferTextureClient(aAllocator, aFormat,
                               aTextureFlags, aMoz2DBackend);
   if (texture) {
     if (!texture->AllocateForSurface(aSize, aAllocFlags)) {
       return nullptr;
     }
   }
@@ -516,16 +524,20 @@ TextureClient::CreateForYCbCr(ISurfaceAl
                               StereoMode aStereoMode,
                               TextureFlags aTextureFlags)
 {
   MOZ_ASSERT(aAllocator->IPCOpen());
   if (!aAllocator || !aAllocator->IPCOpen()) {
     return nullptr;
   }
 
+  if (!gfx::Factory::AllowedSurfaceSize(aYSize)) {
+    return nullptr;
+  }
+
   RefPtr<BufferTextureClient> texture;
   if (aAllocator->IsSameProcess()) {
     texture = new MemoryTextureClient(aAllocator, gfx::SurfaceFormat::YUV,
                                       gfx::BackendType::NONE,
                                       aTextureFlags);
   } else {
     texture = new ShmemTextureClient(aAllocator, gfx::SurfaceFormat::YUV,
                                      gfx::BackendType::NONE,
--- a/gfx/thebes/gfxPlatform.cpp
+++ b/gfx/thebes/gfxPlatform.cpp
@@ -499,23 +499,28 @@ void
 gfxPlatform::Init()
 {
     MOZ_RELEASE_ASSERT(NS_IsMainThread());
     if (gEverInitialized) {
         NS_RUNTIMEABORT("Already started???");
     }
     gEverInitialized = true;
 
-    CrashStatsLogForwarder* logForwarder = new CrashStatsLogForwarder("GraphicsCriticalError");
-    mozilla::gfx::Factory::SetLogForwarder(logForwarder);
-
     // Initialize the preferences by creating the singleton.
     gfxPrefs::GetSingleton();
 
-    logForwarder->SetCircularBufferSize(gfxPrefs::GfxLoggingCrashLength());
+    auto fwd = new CrashStatsLogForwarder("GraphicsCriticalError");
+    fwd->SetCircularBufferSize(gfxPrefs::GfxLoggingCrashLength());
+
+    mozilla::gfx::Config cfg;
+    cfg.mLogForwarder = fwd;
+    cfg.mMaxTextureSize = gfxPrefs::MaxTextureSize();
+    cfg.mMaxAllocSize = gfxPrefs::MaxAllocSize();
+
+    gfx::Factory::Init(cfg);
 
     gGfxPlatformPrefsLock = new Mutex("gfxPlatform::gGfxPlatformPrefsLock");
 
     /* Initialize the GfxInfo service.
      * Note: we can't call functions on GfxInfo that depend
      * on gPlatform until after it has been initialized
      * below. GfxInfo initialization annotates our
      * crash reports so we want to do it before
@@ -696,16 +701,18 @@ gfxPlatform::Shutdown()
 #endif
 
     // This is a bit iffy - we're assuming that we were the ones that set the
     // log forwarder in the Factory, so that it's our responsibility to
     // delete it.
     delete mozilla::gfx::Factory::GetLogForwarder();
     mozilla::gfx::Factory::SetLogForwarder(nullptr);
 
+    gfx::Factory::ShutDown();
+
     delete gGfxPlatformPrefsLock;
 
     gfxPrefs::DestroySingleton();
     gfxFont::DestroySingletons();
 
     delete gPlatform;
     gPlatform = nullptr;
 }
--- a/gfx/thebes/gfxPrefs.h
+++ b/gfx/thebes/gfxPrefs.h
@@ -244,16 +244,19 @@ private:
   DECL_GFX_PREF(Live, "gfx.draw-color-bars",                   CompositorDrawColorBars, bool, false);
   DECL_GFX_PREF(Once, "gfx.e10s.hide-plugins-for-scroll",      HidePluginsForScroll, bool, true);
   DECL_GFX_PREF(Once, "gfx.font_rendering.directwrite.force-enabled", DirectWriteFontRenderingForceEnabled, bool, false);
   DECL_GFX_PREF(Live, "gfx.gralloc.fence-with-readpixels",     GrallocFenceWithReadPixels, bool, false);
   DECL_GFX_PREF(Live, "gfx.layerscope.enabled",                LayerScopeEnabled, bool, false);
   DECL_GFX_PREF(Live, "gfx.layerscope.port",                   LayerScopePort, int32_t, 23456);
   // Note that        "gfx.logging.level" is defined in Logging.h
   DECL_GFX_PREF(Once, "gfx.logging.crash.length",              GfxLoggingCrashLength, uint32_t, 6);
+  // The maximums here are quite conservative, we can tighten them if problems show up.
+  DECL_GFX_PREF(Once, "gfx.max-alloc-size",                    MaxAllocSize, int32_t, (int32_t)500000000);
+  DECL_GFX_PREF(Once, "gfx.max-texture-size",                  MaxTextureSize, int32_t, (int32_t)32767);
   DECL_GFX_PREF(Live, "gfx.perf-warnings.enabled",             PerfWarnings, bool, false);
   DECL_GFX_PREF(Live, "gfx.SurfaceTexture.detach.enabled",     SurfaceTextureDetachEnabled, bool, true);
   DECL_GFX_PREF(Live, "gfx.testing.device-reset",              DeviceResetForTesting, int32_t, 0);
   DECL_GFX_PREF(Live, "gfx.testing.device-fail",               DeviceFailForTesting, bool, false);
 
   // These times should be in milliseconds
   DECL_GFX_PREF(Once, "gfx.touch.resample.delay-threshold",    TouchResampleVsyncDelayThreshold, int32_t, 20);
   DECL_GFX_PREF(Once, "gfx.touch.resample.max-predict",        TouchResampleMaxPredict, int32_t, 8);