Bug 1348941. r=njn, a=lizzard
authorTimothy Nikkel <tnikkel@gmail.com>
Fri, 31 Mar 2017 06:07:29 -0500
changeset 395643 e1e5ac908ce3cab8e0db2f8642f5b98757758a2d
parent 395642 44f3ec83c63dcd88ee9d7dc431237e8228c84779
child 395644 e3ce3282ec26d1cff472f2bed8c65693ea168ff8
push id1468
push userasasaki@mozilla.com
push dateMon, 05 Jun 2017 19:31:07 +0000
treeherdermozilla-release@0641fc6ee9d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnjn, lizzard
bugs1348941
milestone54.0a2
Bug 1348941. r=njn, a=lizzard
image/encoders/bmp/nsBMPEncoder.cpp
image/encoders/bmp/nsBMPEncoder.h
image/encoders/ico/nsICOEncoder.cpp
image/encoders/ico/nsICOEncoder.h
--- a/image/encoders/bmp/nsBMPEncoder.cpp
+++ b/image/encoders/bmp/nsBMPEncoder.cpp
@@ -5,16 +5,17 @@
 
 #include "nsCRT.h"
 #include "mozilla/EndianUtils.h"
 #include "mozilla/UniquePtrExtensions.h"
 #include "nsBMPEncoder.h"
 #include "nsString.h"
 #include "nsStreamUtils.h"
 #include "nsTArray.h"
+#include "mozilla/CheckedInt.h"
 
 using namespace mozilla;
 using namespace mozilla::image;
 using namespace mozilla::image::bmp;
 
 NS_IMPL_ISUPPORTS(nsBMPEncoder, imgIEncoder, nsIInputStream,
                   nsIAsyncInputStream)
 
@@ -53,16 +54,21 @@ nsBMPEncoder::InitFromData(const uint8_t
 {
   // validate input format
   if (aInputFormat != INPUT_FORMAT_RGB &&
       aInputFormat != INPUT_FORMAT_RGBA &&
       aInputFormat != INPUT_FORMAT_HOSTARGB) {
     return NS_ERROR_INVALID_ARG;
   }
 
+  CheckedInt32 check = CheckedInt32(aWidth) * 4;
+  if (MOZ_UNLIKELY(!check.isValid())) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
   // Stride is the padded width of each row, so it better be longer
   if ((aInputFormat == INPUT_FORMAT_RGB &&
        aStride < aWidth * 3) ||
       ((aInputFormat == INPUT_FORMAT_RGBA ||
         aInputFormat == INPUT_FORMAT_HOSTARGB) &&
        aStride < aWidth * 4)) {
       NS_WARNING("Invalid stride for InitFromData");
       return NS_ERROR_INVALID_ARG;
@@ -81,29 +87,29 @@ nsBMPEncoder::InitFromData(const uint8_t
   }
 
   rv = EndImageEncode();
   return rv;
 }
 
 // Just a helper method to make it explicit in calculations that we are dealing
 // with bytes and not bits
-static inline uint32_t
-BytesPerPixel(uint32_t aBPP)
+static inline uint16_t
+BytesPerPixel(uint16_t aBPP)
 {
   return aBPP / 8;
 }
 
 // Calculates the number of padding bytes that are needed per row of image data
 static inline uint32_t
-PaddingBytes(uint32_t aBPP, uint32_t aWidth)
+PaddingBytes(uint16_t aBPP, uint32_t aWidth)
 {
   uint32_t rowSize = aWidth * BytesPerPixel(aBPP);
   uint8_t paddingSize = 0;
-  if(rowSize % 4) {
+  if (rowSize % 4) {
     paddingSize = (4 - (rowSize % 4));
   }
   return paddingSize;
 }
 
 // See ::InitFromData for other info.
 NS_IMETHODIMP
 nsBMPEncoder::StartImageEncode(uint32_t aWidth,
@@ -120,24 +126,31 @@ nsBMPEncoder::StartImageEncode(uint32_t 
   if (aInputFormat != INPUT_FORMAT_RGB &&
       aInputFormat != INPUT_FORMAT_RGBA &&
       aInputFormat != INPUT_FORMAT_HOSTARGB) {
     return NS_ERROR_INVALID_ARG;
   }
 
   // parse and check any provided output options
   Version version;
-  uint32_t bpp;
+  uint16_t bpp;
   nsresult rv = ParseOptions(aOutputOptions, version, bpp);
   if (NS_FAILED(rv)) {
     return rv;
   }
+  MOZ_ASSERT(bpp <= 32);
 
-  InitFileHeader(version, bpp, aWidth, aHeight);
-  InitInfoHeader(version, bpp, aWidth, aHeight);
+  rv = InitFileHeader(version, bpp, aWidth, aHeight);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  rv = InitInfoHeader(version, bpp, aWidth, aHeight);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
 
   mImageBufferSize = mBMPFileHeader.filesize;
   mImageBufferStart = static_cast<uint8_t*>(malloc(mImageBufferSize));
   if (!mImageBufferStart) {
     return NS_ERROR_OUT_OF_MEMORY;
   }
   mImageBufferCurr = mImageBufferStart;
 
@@ -182,22 +195,36 @@ nsBMPEncoder::AddImageFrame(const uint8_
 
   // validate input format
   if (aInputFormat != INPUT_FORMAT_RGB &&
       aInputFormat != INPUT_FORMAT_RGBA &&
       aInputFormat != INPUT_FORMAT_HOSTARGB) {
     return NS_ERROR_INVALID_ARG;
   }
 
-  auto row = MakeUniqueFallible<uint8_t[]>(mBMPInfoHeader.width *
-                                           BytesPerPixel(mBMPInfoHeader.bpp));
+  if (mBMPInfoHeader.width < 0) {
+    return NS_ERROR_ILLEGAL_VALUE;
+  }
+
+  CheckedUint32 size =
+    CheckedUint32(mBMPInfoHeader.width) * CheckedUint32(BytesPerPixel(mBMPInfoHeader.bpp));
+  if (MOZ_UNLIKELY(!size.isValid())) {
+    return NS_ERROR_FAILURE;
+  }
+
+  auto row = MakeUniqueFallible<uint8_t[]>(size.value());
   if (!row) {
     return NS_ERROR_OUT_OF_MEMORY;
   }
 
+  CheckedUint32 check = CheckedUint32(mBMPInfoHeader.height) * aStride;
+  if (MOZ_UNLIKELY(!check.isValid())) {
+    return NS_ERROR_FAILURE;
+  }
+
   // write each row: if we add more input formats, we may want to
   // generalize the conversions
   if (aInputFormat == INPUT_FORMAT_HOSTARGB) {
     // BMP requires RGBA with post-multiplied alpha, so we need to convert
     for (int32_t y = mBMPInfoHeader.height - 1; y >= 0 ; y --) {
       ConvertHostARGBRow(&aData[y * aStride], row, mBMPInfoHeader.width);
       if(mBMPInfoHeader.bpp == 24) {
         EncodeImageDataRow24(row.get());
@@ -251,17 +278,17 @@ nsBMPEncoder::EndImageEncode()
   return NS_OK;
 }
 
 
 // Parses the encoder options and sets the bits per pixel to use
 // See InitFromData for a description of the parse options
 nsresult
 nsBMPEncoder::ParseOptions(const nsAString& aOptions, Version& aVersionOut,
-                           uint32_t& aBppOut)
+                           uint16_t& aBppOut)
 {
   aVersionOut = VERSION_3;
   aBppOut = 24;
 
   // Parse the input string into a set of name/value pairs.
   // From a format like: name=value;bpp=<bpp_value>;name=value
   // to format: [0] = name=value, [1] = bpp=<bpp_value>, [2] = name=value
   nsTArray<nsCString> nameValuePairs;
@@ -419,17 +446,17 @@ nsBMPEncoder::CloseWithStatus(nsresult a
 //    Our colors are stored with premultiplied alphas, but we need
 //    an output with no alpha in machine-independent byte order.
 //
 void
 nsBMPEncoder::ConvertHostARGBRow(const uint8_t* aSrc,
                                  const UniquePtr<uint8_t[]>& aDest,
                                  uint32_t aPixelWidth)
 {
-  int bytes = BytesPerPixel(mBMPInfoHeader.bpp);
+  uint16_t bytes = BytesPerPixel(mBMPInfoHeader.bpp);
 
   if (mBMPInfoHeader.bpp == 32) {
     for (uint32_t x = 0; x < aPixelWidth; x++) {
       const uint32_t& pixelIn = ((const uint32_t*)(aSrc))[x];
       uint8_t* pixelOut = &aDest[x * bytes];
 
       pixelOut[0] = (pixelIn & 0x00ff0000) >> 16;
       pixelOut[1] = (pixelIn & 0x0000ff00) >>  8;
@@ -468,71 +495,104 @@ nsBMPEncoder::NotifyListener()
     mCallbackTarget = nullptr;
     mNotifyThreshold = 0;
 
     callback->OnInputStreamReady(this);
   }
 }
 
 // Initializes the BMP file header mBMPFileHeader to the passed in values
-void
-nsBMPEncoder::InitFileHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
+nsresult
+nsBMPEncoder::InitFileHeader(Version aVersion, uint16_t aBPP, uint32_t aWidth,
                              uint32_t aHeight)
 {
   memset(&mBMPFileHeader, 0, sizeof(mBMPFileHeader));
   mBMPFileHeader.signature[0] = 'B';
   mBMPFileHeader.signature[1] = 'M';
 
   if (aVersion == VERSION_3) {
     mBMPFileHeader.dataoffset = FILE_HEADER_LENGTH + InfoHeaderLength::WIN_V3;
   } else { // aVersion == 5
     mBMPFileHeader.dataoffset = FILE_HEADER_LENGTH + InfoHeaderLength::WIN_V5;
   }
 
   // The color table is present only if BPP is <= 8
   if (aBPP <= 8) {
     uint32_t numColors = 1 << aBPP;
     mBMPFileHeader.dataoffset += 4 * numColors;
-    mBMPFileHeader.filesize = mBMPFileHeader.dataoffset + aWidth * aHeight;
+    CheckedUint32 filesize =
+      CheckedUint32(mBMPFileHeader.dataoffset) + CheckedUint32(aWidth) * aHeight;
+    if (MOZ_UNLIKELY(!filesize.isValid())) {
+      return NS_ERROR_INVALID_ARG;
+    }
+    mBMPFileHeader.filesize = filesize.value();
   } else {
-    mBMPFileHeader.filesize = mBMPFileHeader.dataoffset +
-      (aWidth * BytesPerPixel(aBPP) + PaddingBytes(aBPP, aWidth)) * aHeight;
+    CheckedUint32 filesize =
+      CheckedUint32(mBMPFileHeader.dataoffset) +
+        (CheckedUint32(aWidth) * BytesPerPixel(aBPP) + PaddingBytes(aBPP, aWidth)) * aHeight;
+    if (MOZ_UNLIKELY(!filesize.isValid())) {
+      return NS_ERROR_INVALID_ARG;
+    }
+    mBMPFileHeader.filesize = filesize.value();
   }
 
   mBMPFileHeader.reserved = 0;
+
+  return NS_OK;
 }
 
 #define ENCODE(pImageBufferCurr, value) \
     memcpy(*pImageBufferCurr, &value, sizeof value); \
     *pImageBufferCurr += sizeof value;
 
 // Initializes the bitmap info header mBMPInfoHeader to the passed in values
-void
-nsBMPEncoder::InitInfoHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
+nsresult
+nsBMPEncoder::InitInfoHeader(Version aVersion, uint16_t aBPP, uint32_t aWidth,
                              uint32_t aHeight)
 {
   memset(&mBMPInfoHeader, 0, sizeof(mBMPInfoHeader));
   if (aVersion == VERSION_3) {
     mBMPInfoHeader.bihsize = InfoHeaderLength::WIN_V3;
   } else {
     MOZ_ASSERT(aVersion == VERSION_5);
     mBMPInfoHeader.bihsize = InfoHeaderLength::WIN_V5;
   }
-  mBMPInfoHeader.width = aWidth;
-  mBMPInfoHeader.height = aHeight;
+
+  CheckedInt32 width(aWidth);
+  CheckedInt32 height(aHeight);
+  if (MOZ_UNLIKELY(!width.isValid() || !height.isValid())) {
+    return NS_ERROR_INVALID_ARG;
+  }
+  mBMPInfoHeader.width = width.value();
+  mBMPInfoHeader.height = height.value();
+
   mBMPInfoHeader.planes = 1;
   mBMPInfoHeader.bpp = aBPP;
   mBMPInfoHeader.compression = 0;
   mBMPInfoHeader.colors = 0;
   mBMPInfoHeader.important_colors = 0;
+
+  CheckedUint32 check = CheckedUint32(aWidth) * BytesPerPixel(aBPP);
+  if (MOZ_UNLIKELY(!check.isValid())) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
   if (aBPP <= 8) {
-    mBMPInfoHeader.image_size = aWidth * aHeight;
+    CheckedUint32 imagesize = CheckedUint32(aWidth) * aHeight;
+    if (MOZ_UNLIKELY(!imagesize.isValid())) {
+      return NS_ERROR_INVALID_ARG;
+    }
+    mBMPInfoHeader.image_size = imagesize.value();
   } else {
-    mBMPInfoHeader.image_size =
-      (aWidth * BytesPerPixel(aBPP) + PaddingBytes(aBPP, aWidth)) * aHeight;
+    CheckedUint32 imagesize =
+      (CheckedUint32(aWidth) * BytesPerPixel(aBPP) + PaddingBytes(aBPP, aWidth)) * CheckedUint32(aHeight);
+    if (MOZ_UNLIKELY(!imagesize.isValid())) {
+      return NS_ERROR_INVALID_ARG;
+    }
+    mBMPInfoHeader.image_size = imagesize.value();
   }
   mBMPInfoHeader.xppm = 0;
   mBMPInfoHeader.yppm = 0;
   if (aVersion >= VERSION_5) {
       mBMPInfoHeader.red_mask   = 0x000000FF;
       mBMPInfoHeader.green_mask = 0x0000FF00;
       mBMPInfoHeader.blue_mask  = 0x00FF0000;
       mBMPInfoHeader.alpha_mask = 0xFF000000;
@@ -549,16 +609,18 @@ nsBMPEncoder::InitInfoHeader(Version aVe
       mBMPInfoHeader.gamma_red = 0;
       mBMPInfoHeader.gamma_green = 0;
       mBMPInfoHeader.gamma_blue = 0;
       mBMPInfoHeader.intent = 0;
       mBMPInfoHeader.profile_offset = 0;
       mBMPInfoHeader.profile_size = 0;
       mBMPInfoHeader.reserved = 0;
   }
+
+  return NS_OK;
 }
 
 // Encodes the BMP file header mBMPFileHeader
 void
 nsBMPEncoder::EncodeFileHeader()
 {
   FileHeader littleEndianBFH = mBMPFileHeader;
   NativeEndian::swapToLittleEndianInPlace(&littleEndianBFH.filesize, 1);
--- a/image/encoders/bmp/nsBMPEncoder.h
+++ b/image/encoders/bmp/nsBMPEncoder.h
@@ -99,30 +99,30 @@ protected:
   enum Version
   {
       VERSION_3 = 3,
       VERSION_5 = 5
   };
 
   // See InitData in the cpp for valid parse options
   nsresult ParseOptions(const nsAString& aOptions, Version& aVersionOut,
-                        uint32_t& aBppOut);
+                        uint16_t& aBppOut);
   // Obtains data with no alpha in machine-independent byte order
   void ConvertHostARGBRow(const uint8_t* aSrc,
                           const mozilla::UniquePtr<uint8_t[]>& aDest,
                           uint32_t aPixelWidth);
   // Thread safe notify listener
   void NotifyListener();
 
   // Initializes the bitmap file header member mBMPFileHeader
-  void InitFileHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
-                      uint32_t aHeight);
+  nsresult InitFileHeader(Version aVersion, uint16_t aBPP, uint32_t aWidth,
+                          uint32_t aHeight);
   // Initializes the bitmap info header member mBMPInfoHeader
-  void InitInfoHeader(Version aVersion, uint32_t aBPP, uint32_t aWidth,
-                      uint32_t aHeight);
+  nsresult InitInfoHeader(Version aVersion, uint16_t aBPP, uint32_t aWidth,
+                          uint32_t aHeight);
 
   // Encodes the bitmap file header member mBMPFileHeader
   void EncodeFileHeader();
   // Encodes the bitmap info header member mBMPInfoHeader
   void EncodeInfoHeader();
   // Encodes a row of image data which does not have alpha data
   void EncodeImageDataRow24(const uint8_t* aData);
   // Encodes a row of image data which does have alpha data
--- a/image/encoders/ico/nsICOEncoder.cpp
+++ b/image/encoders/ico/nsICOEncoder.cpp
@@ -223,20 +223,21 @@ nsICOEncoder::StartImageEncode(uint32_t 
   }
 
   // Icons are only 1 byte, so make sure our bitmap is in range
   if (aWidth > 256 || aHeight > 256) {
     return NS_ERROR_INVALID_ARG;
   }
 
   // parse and check any provided output options
-  uint32_t bpp = 24;
+  uint16_t bpp = 24;
   bool usePNG = true;
   nsresult rv = ParseOptions(aOutputOptions, bpp, usePNG);
   NS_ENSURE_SUCCESS(rv, rv);
+  MOZ_ASSERT(bpp <= 32);
 
   mUsePNG = usePNG;
 
   InitFileHeader();
   // The width and height are stored as 0 when we have a value of 256
   InitInfoHeader(bpp, aWidth == 256 ? 0 : (uint8_t)aWidth,
                  aHeight == 256 ? 0 : (uint8_t)aHeight);
 
@@ -260,17 +261,17 @@ nsICOEncoder::EndImageEncode()
   }
 
   return NS_OK;
 }
 
 // Parses the encoder options and sets the bits per pixel to use and PNG or BMP
 // See InitFromData for a description of the parse options
 nsresult
-nsICOEncoder::ParseOptions(const nsAString& aOptions, uint32_t& aBppOut,
+nsICOEncoder::ParseOptions(const nsAString& aOptions, uint16_t& aBppOut,
                            bool& aUsePNGOut)
 {
   // If no parsing options just use the default of 24BPP and PNG yes
   if (aOptions.Length() == 0) {
     aUsePNGOut = true;
     aBppOut = 24;
   }
 
@@ -464,17 +465,17 @@ nsICOEncoder::InitFileHeader()
   memset(&mICOFileHeader, 0, sizeof(mICOFileHeader));
   mICOFileHeader.mReserved = 0;
   mICOFileHeader.mType = 1;
   mICOFileHeader.mCount = 1;
 }
 
 // Initializes the icon directory info header mICODirEntry
 void
-nsICOEncoder::InitInfoHeader(uint32_t aBPP, uint8_t aWidth, uint8_t aHeight)
+nsICOEncoder::InitInfoHeader(uint16_t aBPP, uint8_t aWidth, uint8_t aHeight)
 {
   memset(&mICODirEntry, 0, sizeof(mICODirEntry));
   mICODirEntry.mBitCount = aBPP;
   mICODirEntry.mBytesInRes = 0;
   mICODirEntry.mColorCount = 0;
   mICODirEntry.mWidth = aWidth;
   mICODirEntry.mHeight = aHeight;
   mICODirEntry.mImageOffset = ICONFILEHEADERSIZE + ICODIRENTRYSIZE;
--- a/image/encoders/ico/nsICOEncoder.h
+++ b/image/encoders/ico/nsICOEncoder.h
@@ -45,24 +45,24 @@ public:
   uint32_t GetRealHeight() const
   {
     return mICODirEntry.mHeight == 0 ? 256 : mICODirEntry.mHeight;
   }
 
 protected:
   ~nsICOEncoder();
 
-  nsresult ParseOptions(const nsAString& aOptions, uint32_t& aBppOut,
+  nsresult ParseOptions(const nsAString& aOptions, uint16_t& aBppOut,
                         bool& aUsePNGOut);
   void NotifyListener();
 
   // Initializes the icon file header mICOFileHeader
   void InitFileHeader();
   // Initializes the icon directory info header mICODirEntry
-  void InitInfoHeader(uint32_t aBPP, uint8_t aWidth, uint8_t aHeight);
+  void InitInfoHeader(uint16_t aBPP, uint8_t aWidth, uint8_t aHeight);
   // Encodes the icon file header mICOFileHeader
   void EncodeFileHeader();
   // Encodes the icon directory info header mICODirEntry
   void EncodeInfoHeader();
   // Obtains the current offset filled up to for the image buffer
   inline int32_t GetCurrentImageBufferOffset()
   {
     return static_cast<int32_t>(mImageBufferCurr - mImageBufferStart);