Bug 1533527 - Forbid non-size_t calls to malloc and calloc. r=lsalzman
☠☠ backed out by f0eaf5676456 ☠ ☠
authorJeff Gilbert <jgilbert@mozilla.com>
Sat, 09 Mar 2019 17:40:54 +0000
changeset 521274 5f7ea2187fa6
parent 521272 63bd1994e17c
child 521275 f0eaf5676456
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerslsalzman
bugs1533527
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1533527 - Forbid non-size_t calls to malloc and calloc. r=lsalzman Differential Revision: https://phabricator.services.mozilla.com/D22627
dom/canvas/TexUnpackBlob.cpp
dom/canvas/WebGLContextBuffers.cpp
dom/canvas/WebGLTexture.cpp
dom/canvas/WebGLTextureUpload.cpp
dom/canvas/WebGLTypes.h
gfx/2d/Tools.h
--- a/dom/canvas/TexUnpackBlob.cpp
+++ b/dom/canvas/TexUnpackBlob.cpp
@@ -354,17 +354,17 @@ bool TexUnpackBlob::ConvertIfNeeded(
   ////
 
   const auto dstTotalBytes = CheckedUint32(rowCount) * dstStride;
   if (!dstTotalBytes.isValid()) {
     webgl->ErrorOutOfMemory("Calculation failed.");
     return false;
   }
 
-  UniqueBuffer dstBuffer = calloc(1, dstTotalBytes.value());
+  UniqueBuffer dstBuffer = calloc((size_t)1, (size_t)dstTotalBytes.value());
   if (!dstBuffer.get()) {
     webgl->ErrorOutOfMemory("Failed to allocate dest buffer.");
     return false;
   }
   const auto dstBegin = static_cast<uint8_t*>(dstBuffer.get());
 
   ////
 
--- a/dom/canvas/WebGLContextBuffers.cpp
+++ b/dom/canvas/WebGLContextBuffers.cpp
@@ -298,20 +298,23 @@ void WebGLContext::BufferDataImpl(GLenum
 void WebGLContext::BufferData(GLenum target, WebGLsizeiptr size, GLenum usage) {
   const FuncScope funcScope(*this, "bufferData");
   if (IsContextLost()) return;
 
   if (!ValidateNonNegative("size", size)) return;
 
   ////
 
-  const UniqueBuffer zeroBuffer(calloc(size, 1));
+  const auto checkedSize = CheckedInt<size_t>(size);
+  if (!checkedSize.isValid()) return ErrorOutOfMemory("size too large for platform.");
+
+  const UniqueBuffer zeroBuffer(calloc(checkedSize.value(), size_t(1)));
   if (!zeroBuffer) return ErrorOutOfMemory("Failed to allocate zeros.");
 
-  BufferDataImpl(target, size_t(size), (const uint8_t*)zeroBuffer.get(), usage);
+  BufferDataImpl(target, checkedSize.value(), (const uint8_t*)zeroBuffer.get(), usage);
 }
 
 void WebGLContext::BufferData(GLenum target,
                               const dom::Nullable<dom::ArrayBuffer>& maybeSrc,
                               GLenum usage) {
   const FuncScope funcScope(*this, "bufferData");
   if (IsContextLost()) return;
 
--- a/dom/canvas/WebGLTexture.cpp
+++ b/dom/canvas/WebGLTexture.cpp
@@ -581,17 +581,17 @@ static bool ZeroTextureData(const WebGLC
     checkedByteCount *= widthBlocks;
     checkedByteCount *= heightBlocks;
     checkedByteCount *= depth;
 
     if (!checkedByteCount.isValid()) return false;
 
     const size_t byteCount = checkedByteCount.value();
 
-    UniqueBuffer zeros = calloc(1, byteCount);
+    UniqueBuffer zeros = calloc(size_t(1), byteCount);
     if (!zeros) return false;
 
     ScopedUnpackReset scopedReset(webgl);
     gl->fPixelStorei(LOCAL_GL_UNPACK_ALIGNMENT, 1);  // Don't bother with
                                                      // striding it well.
 
     const auto error =
         DoCompressedTexSubImage(gl, target.get(), level, 0, 0, 0, width, height,
@@ -618,17 +618,17 @@ static bool ZeroTextureData(const WebGLC
   checkedByteCount *= width;
   checkedByteCount *= height;
   checkedByteCount *= depth;
 
   if (!checkedByteCount.isValid()) return false;
 
   const size_t byteCount = checkedByteCount.value();
 
-  UniqueBuffer zeros = calloc(1, byteCount);
+  UniqueBuffer zeros = calloc(size_t(1), byteCount);
   if (!zeros) return false;
 
   ScopedUnpackReset scopedReset(webgl);
   gl->fPixelStorei(LOCAL_GL_UNPACK_ALIGNMENT,
                    1);  // Don't bother with striding it well.
   const auto error = DoTexSubImage(gl, target, level, 0, 0, 0, width, height,
                                    depth, packing, zeros.get());
   return !error;
--- a/dom/canvas/WebGLTextureUpload.cpp
+++ b/dom/canvas/WebGLTextureUpload.cpp
@@ -1946,22 +1946,22 @@ static bool DoCopyTexOrSubImage(WebGLCon
   nsCString errorText;
   do {
     const auto& idealUnpack = dstUsage->idealUnpack;
     if (!isSubImage) {
       UniqueBuffer buffer;
 
       if (uint32_t(rwWidth) != dstWidth || uint32_t(rwHeight) != dstHeight) {
         const auto& pi = idealUnpack->ToPacking();
-        CheckedUint32 byteCount = BytesPerPixel(pi);
+        CheckedInt<size_t> byteCount = BytesPerPixel(pi);
         byteCount *= dstWidth;
         byteCount *= dstHeight;
 
         if (byteCount.isValid()) {
-          buffer = calloc(1, byteCount.value());
+          buffer = calloc(size_t(1), byteCount.value());
         }
 
         if (!buffer.get()) {
           webgl->ErrorOutOfMemory("Ran out of memory allocating zeros.");
           return false;
         }
       }
 
--- a/dom/canvas/WebGLTypes.h
+++ b/dom/canvas/WebGLTypes.h
@@ -10,16 +10,27 @@
 #include "GLTypes.h"
 
 // Manual reflection of WebIDL typedefs that are different from their
 // OpenGL counterparts.
 typedef int64_t WebGLsizeiptr;
 typedef int64_t WebGLintptr;
 typedef bool WebGLboolean;
 
+// -
+// Prevent implicit conversions into calloc and malloc.
+
+template<typename RequireSizeT1, typename RequireSizeT2>
+void* calloc(RequireSizeT1 n, RequireSizeT2 size) = delete;
+
+template<typename RequireSizeT>
+void* malloc(RequireSizeT size) = delete;
+
+// -
+
 namespace mozilla {
 namespace gl {
 class GLContext;  // This is going to be needed a lot.
 }  // namespace gl
 
 /*
  * Implementing WebGL (or OpenGL ES 2.0) on top of desktop OpenGL requires
  * emulating the vertex attrib 0 array when it's not enabled. Indeed,
--- a/gfx/2d/Tools.h
+++ b/gfx/2d/Tools.h
@@ -120,17 +120,17 @@ struct AlignedArray {
       mCount = 0;
       return;
     }
     // We don't create an array of T here, since we don't want ctors to be
     // invoked at the wrong places if we realign below.
     if (aZero) {
       // calloc can be more efficient than new[] for large chunks,
       // so we use calloc/malloc/free for everything.
-      mStorage = static_cast<uint8_t *>(calloc(1, storageByteCount.value()));
+      mStorage = static_cast<uint8_t *>(calloc((size_t)1, storageByteCount.value()));
     } else {
       mStorage = static_cast<uint8_t *>(malloc(storageByteCount.value()));
     }
     if (!mStorage) {
       mStorage = nullptr;
       mPtr = nullptr;
       mCount = 0;
       return;