Bug 1463244 - Cleanup of swizzle stride calculations. r=rhunt, a=RyanVM
MozReview-Commit-ID: GMXRKnu8zHB
--- a/gfx/2d/DataSurfaceHelpers.cpp
+++ b/gfx/2d/DataSurfaceHelpers.cpp
@@ -152,37 +152,44 @@ UniquePtr<uint8_t[]>
SurfaceToPackedBGRA(DataSourceSurface *aSurface)
{
SurfaceFormat format = aSurface->GetFormat();
if (format != SurfaceFormat::B8G8R8A8 && format != SurfaceFormat::B8G8R8X8) {
return nullptr;
}
IntSize size = aSurface->GetSize();
-
- UniquePtr<uint8_t[]> imageBuffer(
- new (std::nothrow) uint8_t[size.width * size.height * sizeof(uint32_t)]);
+ if (size.width < 0 || size.width >= INT32_MAX / 4) {
+ return nullptr;
+ }
+ int32_t stride = size.width * 4;
+ CheckedInt<size_t> bufferSize =
+ CheckedInt<size_t>(stride) * CheckedInt<size_t>(size.height);
+ if (!bufferSize.isValid()) {
+ return nullptr;
+ }
+ UniquePtr<uint8_t[]> imageBuffer(new (std::nothrow) uint8_t[bufferSize.value()]);
if (!imageBuffer) {
return nullptr;
}
DataSourceSurface::MappedSurface map;
if (!aSurface->Map(DataSourceSurface::MapType::READ, &map)) {
return nullptr;
}
CopySurfaceDataToPackedArray(map.mData, imageBuffer.get(), size,
- map.mStride, 4 * sizeof(uint8_t));
+ map.mStride, 4);
aSurface->Unmap();
if (format == SurfaceFormat::B8G8R8X8) {
// Convert BGRX to BGRA by setting a to 255.
- SwizzleData(imageBuffer.get(), size.width * sizeof(uint32_t), SurfaceFormat::X8R8G8B8_UINT32,
- imageBuffer.get(), size.width * sizeof(uint32_t), SurfaceFormat::A8R8G8B8_UINT32,
+ SwizzleData(imageBuffer.get(), stride, SurfaceFormat::X8R8G8B8_UINT32,
+ imageBuffer.get(), stride, SurfaceFormat::A8R8G8B8_UINT32,
size);
}
return imageBuffer;
}
uint8_t*
SurfaceToPackedBGR(DataSourceSurface *aSurface)
@@ -191,30 +198,38 @@ SurfaceToPackedBGR(DataSourceSurface *aS
MOZ_ASSERT(format == SurfaceFormat::B8G8R8X8, "Format not supported");
if (format != SurfaceFormat::B8G8R8X8) {
// To support B8G8R8A8 we'd need to un-pre-multiply alpha
return nullptr;
}
IntSize size = aSurface->GetSize();
-
- uint8_t* imageBuffer = new (std::nothrow) uint8_t[size.width * size.height * 3 * sizeof(uint8_t)];
+ if (size.width < 0 || size.width >= INT32_MAX / 3) {
+ return nullptr;
+ }
+ int32_t stride = size.width * 3;
+ CheckedInt<size_t> bufferSize =
+ CheckedInt<size_t>(stride) * CheckedInt<size_t>(size.height);
+ if (!bufferSize.isValid()) {
+ return nullptr;
+ }
+ uint8_t* imageBuffer = new (std::nothrow) uint8_t[bufferSize.value()];
if (!imageBuffer) {
return nullptr;
}
DataSourceSurface::MappedSurface map;
if (!aSurface->Map(DataSourceSurface::MapType::READ, &map)) {
delete [] imageBuffer;
return nullptr;
}
SwizzleData(map.mData, map.mStride, SurfaceFormat::B8G8R8X8,
- imageBuffer, size.width * 3, SurfaceFormat::B8G8R8,
+ imageBuffer, stride, SurfaceFormat::B8G8R8,
size);
aSurface->Unmap();
return imageBuffer;
}
void
--- a/gfx/2d/Swizzle.cpp
+++ b/gfx/2d/Swizzle.cpp
@@ -254,38 +254,52 @@ PremultiplyFallback(const uint8_t* aSrc,
// If rows are tightly packed, and the size of the total area will fit within
// the precision range of a single row, then process all the data as if it was
// a single row.
static inline IntSize
CollapseSize(const IntSize& aSize, int32_t aSrcStride, int32_t aDstStride)
{
if (aSrcStride == aDstStride &&
- aSrcStride == 4 * aSize.width) {
+ (aSrcStride & 3) == 0 &&
+ aSrcStride / 4 == aSize.width) {
CheckedInt32 area = CheckedInt32(aSize.width) * CheckedInt32(aSize.height);
if (area.isValid()) {
return IntSize(area.value(), 1);
}
}
return aSize;
}
+static inline int32_t
+GetStrideGap(int32_t aWidth, SurfaceFormat aFormat, int32_t aStride)
+{
+ CheckedInt32 used = CheckedInt32(aWidth) * BytesPerPixel(aFormat);
+ if (!used.isValid() || used.value() < 0) {
+ return -1;
+ }
+ return aStride - used.value();
+}
+
bool
PremultiplyData(const uint8_t* aSrc, int32_t aSrcStride, SurfaceFormat aSrcFormat,
uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat,
const IntSize& aSize)
{
if (aSize.IsEmpty()) {
return true;
}
IntSize size = CollapseSize(aSize, aSrcStride, aDstStride);
// Find gap from end of row to the start of the next row.
- int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width;
- int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width;
+ int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride);
+ int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride);
MOZ_ASSERT(srcGap >= 0 && dstGap >= 0);
+ if (srcGap < 0 || dstGap < 0) {
+ return false;
+ }
#define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size)
#ifdef USE_SSE2
switch (FORMAT_KEY(aSrcFormat, aDstFormat)) {
PREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::B8G8R8A8)
PREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::B8G8R8X8)
PREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8A8)
@@ -399,19 +413,22 @@ UnpremultiplyData(const uint8_t* aSrc, i
uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat,
const IntSize& aSize)
{
if (aSize.IsEmpty()) {
return true;
}
IntSize size = CollapseSize(aSize, aSrcStride, aDstStride);
// Find gap from end of row to the start of the next row.
- int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width;
- int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width;
+ int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride);
+ int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride);
MOZ_ASSERT(srcGap >= 0 && dstGap >= 0);
+ if (srcGap < 0 || dstGap < 0) {
+ return false;
+ }
#define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size)
#ifdef USE_SSE2
switch (FORMAT_KEY(aSrcFormat, aDstFormat)) {
UNPREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::B8G8R8A8)
UNPREMULTIPLY_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8A8)
UNPREMULTIPLY_SSE2(SurfaceFormat::R8G8B8A8, SurfaceFormat::R8G8B8A8)
@@ -697,19 +714,22 @@ SwizzleData(const uint8_t* aSrc, int32_t
uint8_t* aDst, int32_t aDstStride, SurfaceFormat aDstFormat,
const IntSize& aSize)
{
if (aSize.IsEmpty()) {
return true;
}
IntSize size = CollapseSize(aSize, aSrcStride, aDstStride);
// Find gap from end of row to the start of the next row.
- int32_t srcGap = aSrcStride - BytesPerPixel(aSrcFormat) * aSize.width;
- int32_t dstGap = aDstStride - BytesPerPixel(aDstFormat) * aSize.width;
+ int32_t srcGap = GetStrideGap(aSize.width, aSrcFormat, aSrcStride);
+ int32_t dstGap = GetStrideGap(aSize.width, aDstFormat, aDstStride);
MOZ_ASSERT(srcGap >= 0 && dstGap >= 0);
+ if (srcGap < 0 || dstGap < 0) {
+ return false;
+ }
#define FORMAT_CASE_CALL(...) __VA_ARGS__(aSrc, srcGap, aDst, dstGap, size)
#ifdef USE_SSE2
switch (FORMAT_KEY(aSrcFormat, aDstFormat)) {
SWIZZLE_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8A8)
SWIZZLE_SSE2(SurfaceFormat::B8G8R8X8, SurfaceFormat::R8G8B8X8)
SWIZZLE_SSE2(SurfaceFormat::B8G8R8A8, SurfaceFormat::R8G8B8X8)