Bug 1348894 - Use CheckedInt more. r=jgilbert a=jcristau
authorL. David Baron <dbaron@dbaron.org>
Wed, 29 Mar 2017 22:52:12 -0400
changeset 379344 8c06f4260c2e25672f4af0f3189a4516a4b3a411
parent 379343 d39a2eac4b5c7b8ec95b65dde91dc541da2f6104
child 379345 21f7a3a22c61534f819419745bd02bf6c65cd6a8
push id1419
push userjlund@mozilla.com
push dateMon, 10 Apr 2017 20:44:07 +0000
treeherdermozilla-release@5e6801b73ef6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgilbert, jcristau
bugs1348894
milestone53.0
Bug 1348894 - Use CheckedInt more. r=jgilbert a=jcristau MozReview-Commit-ID: G5edsCAvRix
dom/media/android/AndroidMediaReader.cpp
gfx/layers/ImageContainer.cpp
--- a/dom/media/android/AndroidMediaReader.cpp
+++ b/dom/media/android/AndroidMediaReader.cpp
@@ -10,16 +10,17 @@
 #include "VideoUtils.h"
 #include "AndroidMediaDecoder.h"
 #include "AndroidMediaPluginHost.h"
 #include "MediaDecoderStateMachine.h"
 #include "ImageContainer.h"
 #include "AbstractMediaDecoder.h"
 #include "gfx2DGlue.h"
 #include "VideoFrameContainer.h"
+#include "mozilla/CheckedInt.h"
 
 namespace mozilla {
 
 using namespace mozilla::gfx;
 using namespace mozilla::media;
 
 typedef mozilla::layers::Image Image;
 typedef mozilla::layers::PlanarYCbCrImage PlanarYCbCrImage;
@@ -389,27 +390,40 @@ AndroidMediaReader::ImageBufferCallback:
   RefPtr<PlanarYCbCrImage> yuvImage = mImageContainer->CreatePlanarYCbCrImage();
   mImage = yuvImage;
 
   if (!yuvImage) {
     NS_WARNING("Could not create I420 image");
     return nullptr;
   }
 
-  size_t frameSize = aWidth * aHeight;
+  // Use uint32_t throughout to match AllocateAndGetNewBuffer's param
+  const auto checkedFrameSize =
+    CheckedInt<uint32_t>(aWidth) * aHeight;
 
   // Allocate enough for one full resolution Y plane
   // and two quarter resolution Cb/Cr planes.
-  uint8_t *buffer = yuvImage->AllocateAndGetNewBuffer(frameSize * 3 / 2);
+  const auto checkedBufferSize =
+    checkedFrameSize + checkedFrameSize / 2;
+
+  if (!checkedBufferSize.isValid()) { // checks checkedFrameSize too
+    NS_WARNING("Could not create I420 image");
+    return nullptr;
+  }
+
+  const auto frameSize = checkedFrameSize.value();
+
+  uint8_t *buffer =
+    yuvImage->AllocateAndGetNewBuffer(checkedBufferSize.value());
 
   mozilla::layers::PlanarYCbCrData frameDesc;
 
   frameDesc.mYChannel = buffer;
   frameDesc.mCbChannel = buffer + frameSize;
-  frameDesc.mCrChannel = buffer + frameSize * 5 / 4;
+  frameDesc.mCrChannel = frameDesc.mCbChannel + frameSize / 4;
 
   frameDesc.mYSize = IntSize(aWidth, aHeight);
   frameDesc.mCbCrSize = IntSize(aWidth / 2, aHeight / 2);
 
   frameDesc.mYStride = aWidth;
   frameDesc.mCbCrStride = aWidth / 2;
 
   frameDesc.mYSkip = 0;
--- a/gfx/layers/ImageContainer.cpp
+++ b/gfx/layers/ImageContainer.cpp
@@ -20,16 +20,17 @@
 #include "mozilla/layers/SharedPlanarYCbCrImage.h"
 #include "mozilla/layers/SharedRGBImage.h"
 #include "mozilla/layers/TextureClientRecycleAllocator.h"
 #include "mozilla/gfx/gfxVars.h"
 #include "nsISupportsUtils.h"           // for NS_IF_ADDREF
 #include "YCbCrUtils.h"                 // for YCbCr conversions
 #include "gfx2DGlue.h"
 #include "mozilla/gfx/2D.h"
+#include "mozilla/CheckedInt.h"
 
 #ifdef XP_MACOSX
 #include "mozilla/gfx/QuartzSupport.h"
 #endif
 
 #ifdef XP_WIN
 #include "gfxWindowsPlatform.h"
 #include <d3d10_1.h>
@@ -483,18 +484,25 @@ CopyPlane(uint8_t *aDst, const uint8_t *
 }
 
 bool
 RecyclingPlanarYCbCrImage::CopyData(const Data& aData)
 {
   mData = aData;
 
   // update buffer size
-  size_t size = mData.mCbCrStride * mData.mCbCrSize.height * 2 +
-                mData.mYStride * mData.mYSize.height;
+  // Use uint32_t throughout to match AllocateBuffer's param and mBufferSize
+  const auto checkedSize =
+    CheckedInt<uint32_t>(mData.mCbCrStride) * mData.mCbCrSize.height * 2 +
+    CheckedInt<uint32_t>(mData.mYStride) * mData.mYSize.height;
+
+  if (!checkedSize.isValid())
+    return false;
+
+  const auto size = checkedSize.value();
 
   // get new buffer
   mBuffer = AllocateBuffer(size);
   if (!mBuffer)
     return false;
 
   // update buffer size
   mBufferSize = size;
@@ -687,18 +695,25 @@ NVImage::AsNVImage()
 
 bool
 NVImage::SetData(const Data& aData)
 {
   MOZ_ASSERT(aData.mCbSkip == 1 && aData.mCrSkip == 1);
   MOZ_ASSERT((int)std::abs(aData.mCbChannel - aData.mCrChannel) == 1);
 
   // Calculate buffer size
-  const uint32_t size = aData.mYSize.height * aData.mYStride +
-                        aData.mCbCrSize.height * aData.mCbCrStride;
+  // Use uint32_t throughout to match AllocateBuffer's param and mBufferSize
+  const auto checkedSize =
+    CheckedInt<uint32_t>(aData.mYSize.height) * aData.mYStride +
+    CheckedInt<uint32_t>(aData.mCbCrSize.height) * aData.mCbCrStride;
+
+  if (!checkedSize.isValid())
+    return false;
+
+  const auto size = checkedSize.value();
 
   // Allocate a new buffer.
   mBuffer = AllocateBuffer(size);
   if (!mBuffer) {
     return false;
   }
 
   // Update mBufferSize.