Protect against integer overflows in the VertexBuffer class by validating the reserved space.

src/libGLESv2/renderer/VertexBuffer.cpp

@@ -107,9 +107,18 @@ int VertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attr
     return oldWritePos;
 }
 
-void VertexBufferInterface::reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances)
+bool VertexBufferInterface::reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances)
 {
-    mReservedSpace += mVertexBuffer->getSpaceRequired(attribute, count, instances);
+    unsigned int requiredSpace = mVertexBuffer->getSpaceRequired(attribute, count, instances);
+
+    // Protect against integer overflow
+    if (mReservedSpace + requiredSpace < mReservedSpace)
+    {
+         return false;
+    }
+
+    mReservedSpace += requiredSpace;
+    return true;
 }
 
 VertexBuffer* VertexBufferInterface::getVertexBuffer() const

src/libGLESv2/renderer/VertexBuffer.h

@@ -60,7 +60,7 @@ class VertexBufferInterface
     VertexBufferInterface(rx::Renderer *renderer, bool dynamic);
     virtual ~VertexBufferInterface();
 
-    void reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances);
+    bool reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances);
 
     unsigned int getBufferSize() const;
 

src/libGLESv2/renderer/VertexDataManager.cpp

@@ -121,12 +121,18 @@ GLenum VertexDataManager::prepareVertexData(const gl::VertexAttribute attribs[],
                     if (staticBuffer->getBufferSize() == 0)
                     {
                         int totalCount = ElementsInBuffer(attribs[i], buffer->size());
-                        staticBuffer->reserveVertexSpace(attribs[i], totalCount, 0);
+                        if (!staticBuffer->reserveVertexSpace(attribs[i], totalCount, 0))
+                        {
+                            return GL_OUT_OF_MEMORY;
+                        }
                     }
                 }
                 else
                 {
-                    mStreamingBuffer->reserveVertexSpace(attribs[i], count, instances);
+                    if (!mStreamingBuffer->reserveVertexSpace(attribs[i], count, instances))
+                    {
+                        return GL_OUT_OF_MEMORY;
+                    }
                 }
             }
         }
@@ -218,7 +224,11 @@ GLenum VertexDataManager::prepareVertexData(const gl::VertexAttribute attribs[],
 
                 if (memcmp(&mCurrentValue[i], &currentValues[i], sizeof(gl::VertexAttribCurrentValueData)) != 0)
                 {
-                    buffer->reserveVertexSpace(attribs[i], 1, 0);
+                    if (!buffer->reserveVertexSpace(attribs[i], 1, 0))
+                    {
+                        return GL_OUT_OF_MEMORY;
+                    }
+
                     int streamOffset = buffer->storeVertexAttributes(attribs[i], currentValues[i], 0, 1, 0);
                     if (streamOffset == -1)
                     {