Bug 1463244 - Cleanup of swizzle stride calculations. r=rhunt, a=RyanVM
authorLee Salzman <lsalzman@mozilla.com>
Fri, 25 May 2018 00:56:22 -0400
changeset 802252 78c708386b35af4145f0f1ba9d1da389e73143b0
parent 802251 0face289ea908187ee19c9400f86f11ee9bf54be
child 802253 9dc397f70f082db1fbdd7c9f8e58bccd3e12da8b
push id111850
push userbmo:tom@mozilla.com
push dateThu, 31 May 2018 16:41:37 +0000
reviewersrhunt, RyanVM
bugs1463244
milestone60.0.2
Bug 1463244 - Cleanup of swizzle stride calculations. r=rhunt, a=RyanVM MozReview-Commit-ID: GMXRKnu8zHB
gfx/2d/DataSurfaceHelpers.cpp
gfx/2d/Swizzle.cpp
--- a/gfx/2d/DataSurfaceHelpers.cpp
+++ b/gfx/2d/DataSurfaceHelpers.cpp
@@ -152,37 +152,44 @@ UniquePtr<uint8_t[]>
 SurfaceToPackedBGRA(DataSourceSurface *aSurface)
 {
   SurfaceFormat format = aSurface->GetFormat();
   if (format != SurfaceFormat::B8G8R8A8 && format != SurfaceFormat::B8G8R8X8) {
     return nullptr;
   }
 
   IntSize size = aSurface->GetSize();
-
-  UniquePtr<uint8_t[]> imageBuffer(
-    new (std::nothrow) uint8_t[size.width * size.height * sizeof(uint32_t)]);
+  if (size.width < 0 || size.width >= INT32_MAX / 4) {
+    return nullptr;
+  }
+  int32_t stride = size.width * 4;
+  CheckedInt<size_t> bufferSize =
+    CheckedInt<size_t>(stride) * CheckedInt<size_t>(size.height);
+  if (!bufferSize.isValid()) {
+    return nullptr;
+  }
+  UniquePtr<uint8_t[]> imageBuffer(new (std::nothrow) uint8_t[bufferSize.value()]);
   if (!imageBuffer) {
     return nullptr;
   }
 
   DataSourceSurface::MappedSurface map;
   if (!aSurface->Map(DataSourceSurface::MapType::READ, &map)) {
     return nullptr;
   }
 
   CopySurfaceDataToPackedArray(map.mData, imageBuffer.get(), size,
-                               map.mStride, 4 * sizeof(uint8_t));
+                               map.mStride, 4);
 
   aSurface->Unmap();
 
   if (format == SurfaceFormat::B8G8R8X8) {
     // Convert BGRX to BGRA by setting a to 255.
-    SwizzleData(imageBuffer.get(), size.width * sizeof(uint32_t), SurfaceFormat::X8R8G8B8_UINT32,
-                imageBuffer.get(), size.width * sizeof(uint32_t), SurfaceFormat::A8R8G8B8_UINT32,
+    SwizzleData(imageBuffer.get(), stride, SurfaceFormat::X8R8G8B8_UINT32,
+                imageBuffer.get(), stride, SurfaceFormat::A8R8G8B8_UINT32,
                 size);
   }
 
   return imageBuffer;
 }
 
 uint8_t*
 SurfaceToPackedBGR(DataSourceSurface *aSurface)
@@ -191,30 +198,38 @@ SurfaceToPackedBGR(DataSourceSurface *aS
   MOZ_ASSERT(format == SurfaceFormat::B8G8R8X8, "Format not supported");
 
   if (format != SurfaceFormat::B8G8R8X8) {
     // To support B8G8R8A8 we'd need to un-pre-multiply alpha
     return nullptr;
   }
 
   IntSize size = aSurface->GetSize();
-
-  uint8_t* imageBuffer = new (std::nothrow) uint8_t[size.width * size.height * 3 * sizeof(uint8_t)];
+  if (size.width < 0 || size.width >= INT32_MAX / 3) {
+    return nullptr;
+  }
+  int32_t stride = size.width * 3;
+  CheckedInt<size_t> bufferSize =
+    CheckedInt<size_t>(stride) * CheckedInt<size_t>(size.height);
+  if (!bufferSize.isValid()) {
+    return nullptr;
+  }
+  uint8_t* imageBuffer = new (std::nothrow) uint8_t[bufferSize.value()];
   if (!imageBuffer) {
     return nullptr;
   }
 
   DataSourceSurface::MappedSurface map;
   if (!aSurface->Map(DataSourceSurface::MapType::READ, &map)) {
     delete [] imageBuffer;
     return nullptr;
   }
 
   SwizzleData(map.mData, map.mStride, SurfaceFormat::B8G8R8X8,
-              imageBuffer, size.width * 3, SurfaceFormat::B8G8R8,
+              imageBuffer, stride, SurfaceFormat::B8G8R8,
               size);
 
   aSurface->Unmap();
 
   return imageBuffer;
 }
 
 void
--- a/gfx/2d/Swizzle.cpp
+++ b/gfx/2d/Swizzle.cpp
@@ -254,38 +254,52 @@ PremultiplyFallback(const uint8_t* aSrc,
 
 // If rows are tightly packed, and the size of the total area will fit within
 // the precision range of a single row, then process all the data as if it was
 // a single row.
 static inline IntSize
 CollapseSize(const IntSize& aSize, int32_t aSrcStride, int32_t aDstStride)
 {
   if (aSrcStride == aDstStride &&
-      aSrcStride == 4 * aSize.width) {
+      (aSrcStride & 3) == 0 &&
+      aSrcStride / 4 == aSize.width) {
     CheckedInt32 area = CheckedInt32(aSize.width) * CheckedInt32(aSize.height);
     if (area.isValid()) {
       return IntSize(area.value(), 1);
     }
   }
   return aSize;
 }
 
+static inline int32_t
+GetStrideGap(int32_t aWidth, SurfaceFormat aFormat, int32_t aStride)
+{
+  CheckedInt32 used = CheckedInt32(aWidth) * BytesPerPixel(aFormat);
+  if (!used.isValid() || used.value() < 0) {
+    return -1;
+  }
+  return aStride - used.value();
+}
+
 bool
 PremultiplyData(const uint8_t* aSrc, int32_t aSrcStride, SurfaceFormat aSrcFormat,
                 uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat,
                 const IntSize& aSize)
 {
   if (aSize.IsEmpty()) {
     return true;
   }
   IntSize size = CollapseSize(aSize, aSrcStride, aDstStride);
   // Find gap from end of row to the start of the next row.
-  int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width;
-  int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width;
+  int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride);
+  int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride);
   MOZ_ASSERT(srcGap >= 0 && dstGap >= 0);
+  if (srcGap < 0 || dstGap < 0) {
+    return false;
+  }
 
 #define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size)
 
 #ifdef USE_SSE2
   switch (FORMAT_KEY(aSrcFormat, aDstFormat)) {
   PREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::B8G8R8A8)
   PREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::B8G8R8X8)
   PREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8A8)
@@ -399,19 +413,22 @@ UnpremultiplyData(const uint8_t* aSrc, i
                   uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat,
                   const IntSize& aSize)
 {
   if (aSize.IsEmpty()) {
     return true;
   }
   IntSize size = CollapseSize(aSize, aSrcStride, aDstStride);
   // Find gap from end of row to the start of the next row.
-  int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width;
-  int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width;
+  int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride);
+  int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride);
   MOZ_ASSERT(srcGap >= 0 && dstGap >= 0);
+  if (srcGap < 0 || dstGap < 0) {
+    return false;
+  }
 
 #define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size)
 
 #ifdef USE_SSE2
   switch (FORMAT_KEY(aSrcFormat, aDstFormat)) {
   UNPREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::B8G8R8A8)
   UNPREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8A8)
   UNPREMULTIPLY_SSE2(SurfaceFormat::R8G8B8A8, SurfaceFormat::R8G8B8A8)
@@ -697,19 +714,22 @@ SwizzleData(const uint8_t* aSrc, int32_t
             uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat,
             const IntSize& aSize)
 {
   if (aSize.IsEmpty()) {
     return true;
   }
   IntSize size = CollapseSize(aSize, aSrcStride, aDstStride);
   // Find gap from end of row to the start of the next row.
-  int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width;
-  int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width;
+  int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride);
+  int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride);
   MOZ_ASSERT(srcGap >= 0 && dstGap >= 0);
+  if (srcGap < 0 || dstGap < 0) {
+    return false;
+  }
 
 #define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size)
 
 #ifdef USE_SSE2
   switch (FORMAT_KEY(aSrcFormat, aDstFormat)) {
   SWIZZLE_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8A8)
   SWIZZLE_SSE2(SurfaceFormat::B8G8R8X8, SurfaceFormat::R8G8B8X8)
   SWIZZLE_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8X8)