Bug 1533527 - Forbid non-size_t calls to malloc and calloc. r=lsalzman
authorJeff Gilbert <jgilbert@mozilla.com>
Wed, 06 Mar 2019 16:41:16 -0800
changeset 464741 bd5f8c7b771725361f054b02a4a641820482e1f1
parent 464740 40b1d8d859e2724a978d2db3cf0ae5121688b89d
child 464785 6ed5e5a3e39ec3155a2a36f4d567e927df3a6f39
push id112480
push userjgilbert@mozilla.com
push dateMon, 18 Mar 2019 20:47:27 +0000
treeherdermozilla-inbound@bd5f8c7b7717 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerslsalzman
bugs1533527
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1533527 - Forbid non-size_t calls to malloc and calloc. r=lsalzman Differential Revision: https://phabricator.services.mozilla.com/D22627
dom/canvas/TexUnpackBlob.cpp
dom/canvas/WebGLContextBuffers.cpp
dom/canvas/WebGLTexture.cpp
dom/canvas/WebGLTextureUpload.cpp
dom/canvas/WebGLTypes.h
gfx/2d/Tools.h
--- a/dom/canvas/TexUnpackBlob.cpp
+++ b/dom/canvas/TexUnpackBlob.cpp
@@ -354,17 +354,17 @@ bool TexUnpackBlob::ConvertIfNeeded(
   ////
 
   const auto dstTotalBytes = CheckedUint32(rowCount) * dstStride;
   if (!dstTotalBytes.isValid()) {
     webgl->ErrorOutOfMemory("Calculation failed.");
     return false;
   }
 
-  UniqueBuffer dstBuffer = calloc(1, dstTotalBytes.value());
+  UniqueBuffer dstBuffer = calloc(1u, (size_t)dstTotalBytes.value());
   if (!dstBuffer.get()) {
     webgl->ErrorOutOfMemory("Failed to allocate dest buffer.");
     return false;
   }
   const auto dstBegin = static_cast<uint8_t*>(dstBuffer.get());
 
   ////
 
--- a/dom/canvas/WebGLContextBuffers.cpp
+++ b/dom/canvas/WebGLContextBuffers.cpp
@@ -298,20 +298,23 @@ void WebGLContext::BufferDataImpl(GLenum
 void WebGLContext::BufferData(GLenum target, WebGLsizeiptr size, GLenum usage) {
   const FuncScope funcScope(*this, "bufferData");
   if (IsContextLost()) return;
 
   if (!ValidateNonNegative("size", size)) return;
 
   ////
 
-  const UniqueBuffer zeroBuffer(calloc(size, 1));
+  const auto checkedSize = CheckedInt<size_t>(size);
+  if (!checkedSize.isValid()) return ErrorOutOfMemory("size too large for platform.");
+
+  const UniqueBuffer zeroBuffer(calloc(checkedSize.value(), 1u));
   if (!zeroBuffer) return ErrorOutOfMemory("Failed to allocate zeros.");
 
-  BufferDataImpl(target, size_t(size), (const uint8_t*)zeroBuffer.get(), usage);
+  BufferDataImpl(target, checkedSize.value(), (const uint8_t*)zeroBuffer.get(), usage);
 }
 
 void WebGLContext::BufferData(GLenum target,
                               const dom::Nullable<dom::ArrayBuffer>& maybeSrc,
                               GLenum usage) {
   const FuncScope funcScope(*this, "bufferData");
   if (IsContextLost()) return;
 
--- a/dom/canvas/WebGLTexture.cpp
+++ b/dom/canvas/WebGLTexture.cpp
@@ -581,17 +581,17 @@ static bool ZeroTextureData(const WebGLC
     checkedByteCount *= widthBlocks;
     checkedByteCount *= heightBlocks;
     checkedByteCount *= depth;
 
     if (!checkedByteCount.isValid()) return false;
 
     const size_t byteCount = checkedByteCount.value();
 
-    UniqueBuffer zeros = calloc(1, byteCount);
+    UniqueBuffer zeros = calloc(1u, byteCount);
     if (!zeros) return false;
 
     ScopedUnpackReset scopedReset(webgl);
     gl->fPixelStorei(LOCAL_GL_UNPACK_ALIGNMENT, 1);  // Don't bother with
                                                      // striding it well.
 
     const auto error =
         DoCompressedTexSubImage(gl, target.get(), level, 0, 0, 0, width, height,
@@ -618,17 +618,17 @@ static bool ZeroTextureData(const WebGLC
   checkedByteCount *= width;
   checkedByteCount *= height;
   checkedByteCount *= depth;
 
   if (!checkedByteCount.isValid()) return false;
 
   const size_t byteCount = checkedByteCount.value();
 
-  UniqueBuffer zeros = calloc(1, byteCount);
+  UniqueBuffer zeros = calloc(1u, byteCount);
   if (!zeros) return false;
 
   ScopedUnpackReset scopedReset(webgl);
   gl->fPixelStorei(LOCAL_GL_UNPACK_ALIGNMENT,
                    1);  // Don't bother with striding it well.
   const auto error = DoTexSubImage(gl, target, level, 0, 0, 0, width, height,
                                    depth, packing, zeros.get());
   return !error;
--- a/dom/canvas/WebGLTextureUpload.cpp
+++ b/dom/canvas/WebGLTextureUpload.cpp
@@ -1946,22 +1946,22 @@ static bool DoCopyTexOrSubImage(WebGLCon
   nsCString errorText;
   do {
     const auto& idealUnpack = dstUsage->idealUnpack;
     if (!isSubImage) {
       UniqueBuffer buffer;
 
       if (uint32_t(rwWidth) != dstWidth || uint32_t(rwHeight) != dstHeight) {
         const auto& pi = idealUnpack->ToPacking();
-        CheckedUint32 byteCount = BytesPerPixel(pi);
+        CheckedInt<size_t> byteCount = BytesPerPixel(pi);
         byteCount *= dstWidth;
         byteCount *= dstHeight;
 
         if (byteCount.isValid()) {
-          buffer = calloc(1, byteCount.value());
+          buffer = calloc(1u, byteCount.value());
         }
 
         if (!buffer.get()) {
           webgl->ErrorOutOfMemory("Ran out of memory allocating zeros.");
           return false;
         }
       }
 
--- a/dom/canvas/WebGLTypes.h
+++ b/dom/canvas/WebGLTypes.h
@@ -1,30 +1,62 @@
 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef WEBGLTYPES_H_
 #define WEBGLTYPES_H_
 
+#include <limits>
+
 // Most WebIDL typedefs are identical to their OpenGL counterparts.
 #include "GLTypes.h"
 
 // Manual reflection of WebIDL typedefs that are different from their
 // OpenGL counterparts.
 typedef int64_t WebGLsizeiptr;
 typedef int64_t WebGLintptr;
 typedef bool WebGLboolean;
 
+// -
+
 namespace mozilla {
 namespace gl {
 class GLContext;  // This is going to be needed a lot.
 }  // namespace gl
 
+// -
+// Prevent implicit conversions into calloc and malloc. (mozilla namespace only!)
+
+template<typename DestT>
+class ForbidNarrowing final
+{
+  DestT mVal;
+
+public:
+  template<typename SrcT>
+  MOZ_IMPLICIT ForbidNarrowing(SrcT val) : mVal(val) {
+    static_assert(std::numeric_limits<SrcT>::min() >= std::numeric_limits<DestT>::min(),
+                  "SrcT must be narrower than DestT.");
+    static_assert(std::numeric_limits<SrcT>::max() <= std::numeric_limits<DestT>::max(),
+                  "SrcT must be narrower than DestT.");
+  }
+
+  explicit operator DestT() const { return mVal; }
+};
+
+inline void* malloc(const ForbidNarrowing<size_t> s) {
+  return ::malloc(size_t(s));
+}
+
+inline void* calloc(const ForbidNarrowing<size_t> n, const ForbidNarrowing<size_t> size) {
+  return ::calloc(size_t(n), size_t(size));
+}
+
 /*
  * Implementing WebGL (or OpenGL ES 2.0) on top of desktop OpenGL requires
  * emulating the vertex attrib 0 array when it's not enabled. Indeed,
  * OpenGL ES 2.0 allows drawing without vertex attrib 0 array enabled, but
  * desktop OpenGL does not allow that.
  */
 enum class WebGLVertexAttrib0Status : uint8_t {
   Default,                     // default status - no emulation needed
--- a/gfx/2d/Tools.h
+++ b/gfx/2d/Tools.h
@@ -120,17 +120,17 @@ struct AlignedArray {
       mCount = 0;
       return;
     }
     // We don't create an array of T here, since we don't want ctors to be
     // invoked at the wrong places if we realign below.
     if (aZero) {
       // calloc can be more efficient than new[] for large chunks,
       // so we use calloc/malloc/free for everything.
-      mStorage = static_cast<uint8_t *>(calloc(1, storageByteCount.value()));
+      mStorage = static_cast<uint8_t *>(calloc(1u, storageByteCount.value()));
     } else {
       mStorage = static_cast<uint8_t *>(malloc(storageByteCount.value()));
     }
     if (!mStorage) {
       mStorage = nullptr;
       mPtr = nullptr;
       mCount = 0;
       return;