Bug 1544180 - Add CheckedInt check r=jgilbert
authorsotaro <sotaro.ikeda.g@gmail.com>
Tue, 14 May 2019 12:05:57 +0900
changeset 532615 271512a8bcc3196dfb14e603d39bc57d9c169454
parent 532532 998d83fa2ae988509435ba88de2c640f4607f2fa
child 532616 8df2d171f7fd925474dedb797ac46b21a135e496
push id11270
push userrgurzau@mozilla.com
push dateWed, 15 May 2019 15:07:19 +0000
treeherdermozilla-beta@571bc76da583 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgilbert
bugs1544180
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1544180 - Add CheckedInt check r=jgilbert Differential Revision: https://phabricator.services.mozilla.com/D30849
gfx/gl/GLUploadHelpers.cpp
--- a/gfx/gl/GLUploadHelpers.cpp
+++ b/gfx/gl/GLUploadHelpers.cpp
@@ -139,18 +139,32 @@ static void TexSubImage2DWithUnpackSubim
 static void TexSubImage2DWithoutUnpackSubimage(
     GLContext* gl, GLenum target, GLint level, GLint xoffset, GLint yoffset,
     GLsizei width, GLsizei height, GLsizei stride, GLint pixelsize,
     GLenum format, GLenum type, const GLvoid* pixels) {
   // Not using the whole row of texture data and GL_UNPACK_ROW_LENGTH
   // isn't supported. We make a copy of the texture data we're using,
   // such that we're using the whole row of data in the copy. This turns
   // out to be more efficient than uploading row-by-row; see bug 698197.
-  unsigned char* newPixels =
-      new (fallible) unsigned char[width * height * pixelsize];
+
+  // Width and height are never more than 16384. At 16Ki*16Ki, 4bpp is 1GiB, but
+  // if we allow 8bpp (or higher) here, that's 2GiB, which would overflow on
+  // 32-bit.
+  MOZ_ASSERT(width <= 16384);
+  MOZ_ASSERT(height <= 16384);
+  MOZ_ASSERT(pixelsize < 8);
+
+  const auto size = CheckedInt<size_t>(width) * height * pixelsize;
+  if (!size.isValid()) {
+    // This should never happen, but we use a defensive check.
+    MOZ_ASSERT_UNREACHABLE("Unacceptable size calculated.!");
+    return;
+  }
+
+  unsigned char* newPixels = new (fallible) unsigned char[size.value()];
 
   if (newPixels) {
     unsigned char* rowDest = newPixels;
     const unsigned char* rowSource = (const unsigned char*)pixels;
     for (int h = 0; h < height; h++) {
       memcpy(rowDest, rowSource, width * pixelsize);
       rowDest += width * pixelsize;
       rowSource += stride;
@@ -235,18 +249,32 @@ static void TexImage2DHelper(GLContext* 
     if (!CanUploadNonPowerOfTwo(gl) &&
         (stride != width * pixelsize || !IsPowerOfTwo((uint32_t)width) ||
          !IsPowerOfTwo((uint32_t)height))) {
       // Pad out texture width and height to the next power of two
       // as we don't support/want non power of two texture uploads
       GLsizei paddedWidth = RoundUpPow2((uint32_t)width);
       GLsizei paddedHeight = RoundUpPow2((uint32_t)height);
 
-      GLvoid* paddedPixels =
-          new unsigned char[paddedWidth * paddedHeight * pixelsize];
+      // Width and height are never more than 16384. At 16Ki*16Ki, 4bpp is 1GiB,
+      // but if we allow 8bpp (or higher) here, that's 2GiB, which would
+      // overflow on 32-bit.
+      MOZ_ASSERT(width <= 16384);
+      MOZ_ASSERT(height <= 16384);
+      MOZ_ASSERT(pixelsize < 8);
+
+      const auto size =
+          CheckedInt<size_t>(paddedWidth) * paddedHeight * pixelsize;
+      if (!size.isValid()) {
+        // This should never happen, but we use a defensive check.
+        MOZ_ASSERT_UNREACHABLE("Unacceptable size calculated.!");
+        return;
+      }
+
+      GLvoid* paddedPixels = new unsigned char[size.value()];
 
       // Pad out texture data to be in a POT sized buffer for uploading to
       // a POT sized texture
       CopyAndPadTextureData(pixels, paddedPixels, width, height, paddedWidth,
                             paddedHeight, stride, pixelsize);
 
       gl->fPixelStorei(
           LOCAL_GL_UNPACK_ALIGNMENT,
@@ -378,23 +406,27 @@ SurfaceFormat UploadImageDataToTexture(
       format = LOCAL_GL_LUMINANCE;
       internalFormat = LOCAL_GL_LUMINANCE16;
       type = LOCAL_GL_UNSIGNED_SHORT;
       // We don't have a specific luminance shader
       surfaceFormat = SurfaceFormat::A8;
       pixelSize = 2;
       break;
     default:
-      NS_ASSERTION(false, "Unhandled image surface format!");
+      MOZ_ASSERT_UNREACHABLE("Unhandled image surface format!");
   }
 
   if (aOutUploadSize) {
     *aOutUploadSize = 0;
   }
 
+  if (surfaceFormat == gfx::SurfaceFormat::UNKNOWN) {
+    return gfx::SurfaceFormat::UNKNOWN;
+  }
+
   if (aNeedInit || !CanUploadSubTextures(gl)) {
     // If the texture needs initialized, or we are unable to
     // upload sub textures, then initialize and upload the entire
     // texture.
     TexImage2DHelper(gl, aTextureTarget, 0, internalFormat, aSize.width,
                      aSize.height, aStride, pixelSize, 0, format, type, aData);
 
     if (aOutUploadSize && aNeedInit) {