Bug 1190526 - Check for overflow in vertex allocation. r=jrmuizel, a=dveditz
authorKyle <kfung@mozilla.com>
Mon, 17 Aug 2015 11:31:17 -0400
changeset 283409 b6460380ae8c6c56609f636f42a260c61a4c6f7c
parent 283408 9603c34d323749b9947d17fa54cd6c937999623d
child 283410 f42fa2b6cc429a235ad8cffe905f9cdf06273f59
push id113
push userryanvm@gmail.com
push dateThu, 05 Nov 2015 16:09:15 +0000
reviewersjrmuizel, dveditz
bugs1190526
milestone42.0a2
Bug 1190526 - Check for overflow in vertex allocation. r=jrmuizel, a=dveditz
gfx/angle/src/libGLESv2/renderer/d3d/VertexBuffer.cpp
--- a/gfx/angle/src/libGLESv2/renderer/d3d/VertexBuffer.cpp
+++ b/gfx/angle/src/libGLESv2/renderer/d3d/VertexBuffer.cpp
@@ -96,17 +96,22 @@ gl::Error VertexBufferInterface::storeVe
 
     unsigned int spaceRequired;
     error = mVertexBuffer->getSpaceRequired(attrib, count, instances, &spaceRequired);
     if (error.isError())
     {
         return error;
     }
 
-    if (mWritePosition + spaceRequired < mWritePosition)
+    // Align to 16-byte boundary
+    unsigned int alignedSpaceRequired = roundUp(spaceRequired, 16u);
+
+    // Protect against integer overflow
+    if (!IsUnsignedAdditionSafe(mWritePosition, alignedSpaceRequired) ||
+        alignedSpaceRequired < spaceRequired)
     {
         return gl::Error(GL_OUT_OF_MEMORY, "Internal error, new vertex buffer write position would overflow.");
     }
 
     error = reserveSpace(mReservedSpace);
     if (error.isError())
     {
         return error;
@@ -119,46 +124,44 @@ gl::Error VertexBufferInterface::storeVe
         return error;
     }
 
     if (outStreamOffset)
     {
         *outStreamOffset = mWritePosition;
     }
 
-    mWritePosition += spaceRequired;
-
-    // Align to 16-byte boundary
-    mWritePosition = rx::roundUp(mWritePosition, 16u);
+    mWritePosition += alignedSpaceRequired;
 
     return gl::Error(GL_NO_ERROR);
 }
 
 gl::Error VertexBufferInterface::reserveVertexSpace(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances)
 {
     gl::Error error(GL_NO_ERROR);
 
     unsigned int requiredSpace;
     error = mVertexBuffer->getSpaceRequired(attrib, count, instances, &requiredSpace);
     if (error.isError())
     {
         return error;
     }
 
+    // Align to 16-byte boundary
+    unsigned int alignedRequiredSpace = roundUp(requiredSpace, 16u);
+
     // Protect against integer overflow
-    if (mReservedSpace + requiredSpace < mReservedSpace)
+    if (!IsUnsignedAdditionSafe(mReservedSpace, alignedRequiredSpace) ||
+        alignedRequiredSpace < requiredSpace)
     {
         return gl::Error(GL_OUT_OF_MEMORY, "Unable to reserve %u extra bytes in internal vertex buffer, "
                          "it would result in an overflow.", requiredSpace);
     }
 
-    mReservedSpace += requiredSpace;
-
-    // Align to 16-byte boundary
-    mReservedSpace = rx::roundUp(mReservedSpace, 16u);
+    mReservedSpace += alignedRequiredSpace;
 
     return gl::Error(GL_NO_ERROR);
 }
 
 VertexBuffer* VertexBufferInterface::getVertexBuffer() const
 {
     return mVertexBuffer;
 }