Updated VertexBuffer's getSpaceRequired and storeVertexAttributes methods to…

src/libGLESv2/renderer/IndexBuffer.cpp

@@ -67,18 +67,30 @@ unsigned int IndexBufferInterface::getSerial() const
     return mIndexBuffer->getSerial();
 }
 
-int IndexBufferInterface::mapBuffer(unsigned int size, void** outMappedMemory)
+bool IndexBufferInterface::mapBuffer(unsigned int size, void** outMappedMemory, unsigned int *streamOffset)
 {
+    // Protect against integer overflow
+    if (mWritePosition + size < mWritePosition)
+    {
+        return false;
+    }
+
     if (!mIndexBuffer->mapBuffer(mWritePosition, size, outMappedMemory))
     {
-        *outMappedMemory = NULL;
-        return -1;
+        if (outMappedMemory)
+        {
+            *outMappedMemory = NULL;
+        }
+        return false;
     }
 
-    int oldWritePos = static_cast<int>(mWritePosition);
-    mWritePosition += size;
+    if (streamOffset)
+    {
+        *streamOffset = mWritePosition;
+    }
 
-    return oldWritePos;
+    mWritePosition += size;
+    return true;
 }
 
 bool IndexBufferInterface::unmapBuffer()

src/libGLESv2/renderer/IndexBuffer.h

@@ -59,7 +59,7 @@ class IndexBufferInterface
 
     unsigned int getSerial() const;
 
-    int mapBuffer(unsigned int size, void** outMappedMemory);
+    bool mapBuffer(unsigned int size, void** outMappedMemory, unsigned int *streamOffset);
     bool unmapBuffer();
 
     IndexBuffer *getIndexBuffer() const;

src/libGLESv2/renderer/IndexBuffer11.cpp

@@ -75,7 +75,8 @@ bool IndexBuffer11::mapBuffer(unsigned int offset, unsigned int size, void** out
 {
     if (mBuffer)
     {
-        if (offset + size > mBufferSize)
+        // Check for integer overflows and out-out-bounds map requests
+        if (offset + size < offset || offset + size > mBufferSize)
         {
             ERR("Index buffer map range is not inside the buffer.");
             return false;

src/libGLESv2/renderer/IndexDataManager.cpp

@@ -132,7 +132,16 @@ GLenum IndexDataManager::prepareIndexData(GLenum type, GLsizei count, gl::Buffer
           default: UNREACHABLE(); alignedOffset = false;
         }
 
-        if (gl::ComputeTypeSize(type) * count + offset > static_cast<GLsizei>(storage->getSize()))
+        unsigned int typeSize = gl::ComputeTypeSize(type);
+
+        // check for integer overflows and underflows
+        if (static_cast<unsigned int>(offset) > (std::numeric_limits<unsigned int>::max() / typeSize) ||
+            static_cast<unsigned int>(count) > ((std::numeric_limits<unsigned int>::max() / typeSize) - offset))
+        {
+            return GL_OUT_OF_MEMORY;
+        }
+
+        if (typeSize * static_cast<unsigned int>(count) + offset > storage->getSize())
         {
             return GL_INVALID_OPERATION;
         }
@@ -146,7 +155,7 @@ GLenum IndexDataManager::prepareIndexData(GLenum type, GLsizei count, gl::Buffer
     IndexBufferInterface *indexBuffer = streamingBuffer;
     bool directStorage = alignedOffset && storage && storage->supportsDirectBinding() &&
                          destinationIndexType == type;
-    UINT streamOffset = 0;
+    unsigned int streamOffset = 0;
 
     if (directStorage)
     {
@@ -176,7 +185,7 @@ GLenum IndexDataManager::prepareIndexData(GLenum type, GLsizei count, gl::Buffer
     }
     else
     {
-        int convertCount = count;
+        unsigned int convertCount = count;
 
         if (staticBuffer)
         {
@@ -198,12 +207,22 @@ GLenum IndexDataManager::prepareIndexData(GLenum type, GLsizei count, gl::Buffer
             return GL_INVALID_OPERATION;
         }
 
-        unsigned int bufferSizeRequired = convertCount * gl::ComputeTypeSize(destinationIndexType);
-        indexBuffer->reserveBufferSpace(bufferSizeRequired, type);
+        unsigned int indexTypeSize = gl::ComputeTypeSize(destinationIndexType);
+        if (convertCount > std::numeric_limits<unsigned int>::max() / indexTypeSize)
+        {
+            ERR("Reserving %u indicies of %u bytes each exceeds the maximum buffer size.", convertCount, indexTypeSize);
+            return GL_OUT_OF_MEMORY;
+        }
+
+        unsigned int bufferSizeRequired = convertCount * indexTypeSize;
+        if (!indexBuffer->reserveBufferSpace(bufferSizeRequired, type))
+        {
+            ERR("Failed to reserve %u bytes in an index buffer.", bufferSizeRequired);
+            return GL_OUT_OF_MEMORY;
+        }
 
         void* output = NULL;
-        streamOffset = indexBuffer->mapBuffer(bufferSizeRequired, &output);
-        if (streamOffset == -1 || output == NULL)
+        if (!indexBuffer->mapBuffer(bufferSizeRequired, &output, &streamOffset))
         {
             ERR("Failed to map index buffer.");
             return GL_OUT_OF_MEMORY;
@@ -254,7 +273,7 @@ StaticIndexBufferInterface *IndexDataManager::getCountingIndices(GLsizei count)
             mCountingBuffer->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_SHORT);
 
             void* mappedMemory = NULL;
-            if (mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory) == -1 || mappedMemory == NULL)
+            if (!mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory, NULL))
             {
                 ERR("Failed to map counting buffer.");
                 return NULL;
@@ -284,7 +303,7 @@ StaticIndexBufferInterface *IndexDataManager::getCountingIndices(GLsizei count)
             mCountingBuffer->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_INT);
 
             void* mappedMemory = NULL;
-            if (mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory) == -1 || mappedMemory == NULL)
+            if (!mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory, NULL))
             {
                 ERR("Failed to map counting buffer.");
                 return NULL;

src/libGLESv2/renderer/Renderer11.cpp

@@ -1150,15 +1150,15 @@ void Renderer11::drawLineLoop(GLsizei count, GLenum type, const GLvoid *indices,
     }
 
     void* mappedMemory = NULL;
-    int offset = mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory);
-    if (offset == -1 || mappedMemory == NULL)
+    unsigned int offset;
+    if (!mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
     {
         ERR("Could not map index buffer for GL_LINE_LOOP.");
         return gl::error(GL_OUT_OF_MEMORY);
     }
 
     unsigned int *data = reinterpret_cast<unsigned int*>(mappedMemory);
-    unsigned int indexBufferOffset = static_cast<unsigned int>(offset);
+    unsigned int indexBufferOffset = offset;
 
     switch (type)
     {
@@ -1255,15 +1255,15 @@ void Renderer11::drawTriangleFan(GLsizei count, GLenum type, const GLvoid *indic
     }
 
     void* mappedMemory = NULL;
-    int offset = mTriangleFanIB->mapBuffer(spaceNeeded, &mappedMemory);
-    if (offset == -1 || mappedMemory == NULL)
+    unsigned int offset;
+    if (!mTriangleFanIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
     {
         ERR("Could not map scratch index buffer for GL_TRIANGLE_FAN.");
         return gl::error(GL_OUT_OF_MEMORY);
     }
 
     unsigned int *data = reinterpret_cast<unsigned int*>(mappedMemory);
-    unsigned int indexBufferOffset = static_cast<unsigned int>(offset);
+    unsigned int indexBufferOffset = offset;
 
     switch (type)
     {

src/libGLESv2/renderer/Renderer9.cpp

@@ -1454,7 +1454,7 @@ void Renderer9::drawLineLoop(GLsizei count, GLenum type, const GLvoid *indices,
         indices = static_cast<const GLubyte*>(storage->getData()) + offset;
     }
 
-    UINT startIndex = 0;
+    unsigned int startIndex = 0;
 
     if (get32BitIndexSupport())
     {
@@ -1488,14 +1488,14 @@ void Renderer9::drawLineLoop(GLsizei count, GLenum type, const GLvoid *indices,
         }
 
         void* mappedMemory = NULL;
-        int offset = mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory);
-        if (offset == -1 || mappedMemory == NULL)
+        unsigned int offset = 0;
+        if (!mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
         {
             ERR("Could not map index buffer for GL_LINE_LOOP.");
             return gl::error(GL_OUT_OF_MEMORY);
         }
 
-        startIndex = static_cast<UINT>(offset) / 4;
+        startIndex = static_cast<unsigned int>(offset) / 4;
         unsigned int *data = reinterpret_cast<unsigned int*>(mappedMemory);
 
         switch (type)
@@ -1569,14 +1569,14 @@ void Renderer9::drawLineLoop(GLsizei count, GLenum type, const GLvoid *indices,
         }
 
         void* mappedMemory = NULL;
-        int offset = mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory);
-        if (offset == -1 || mappedMemory == NULL)
+        unsigned int offset;
+        if (mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
         {
             ERR("Could not map index buffer for GL_LINE_LOOP.");
             return gl::error(GL_OUT_OF_MEMORY);
         }
 
-        startIndex = static_cast<UINT>(offset) / 2;
+        startIndex = static_cast<unsigned int>(offset) / 2;
         unsigned short *data = reinterpret_cast<unsigned short*>(mappedMemory);
 
         switch (type)

src/libGLESv2/renderer/VertexBuffer.cpp

@@ -87,47 +87,76 @@ bool VertexBufferInterface::discard()
     return mVertexBuffer->discard();
 }
 
-int VertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances)
+bool VertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib,  GLint start, GLsizei count, GLsizei instances,
+                                                  unsigned int *outStreamOffset)
 {
+    unsigned int spaceRequired;
+    if (!mVertexBuffer->getSpaceRequired(attrib, count, instances, &spaceRequired))
+    {
+        return false;
+    }
+
+    if (mWritePosition + spaceRequired < mWritePosition)
+    {
+        return false;
+    }
+
     if (!reserveSpace(mReservedSpace))
     {
-        return -1;
+        return false;
     }
     mReservedSpace = 0;
 
     if (!mVertexBuffer->storeVertexAttributes(attrib, start, count, instances, mWritePosition))
     {
-        return -1;
+        return false;
     }
 
-    int oldWritePos = static_cast<int>(mWritePosition);
-    mWritePosition += mVertexBuffer->getSpaceRequired(attrib, count, instances);
+    if (outStreamOffset)
+    {
+        *outStreamOffset = mWritePosition;
+    }
+
+    mWritePosition += spaceRequired;
 
-    return oldWritePos;
+    return true;
 }
 
-int VertexBufferInterface::storeRawData(const void* data, unsigned int size)
+bool VertexBufferInterface::storeRawData(const void* data, unsigned int size, unsigned int *outStreamOffset)
 {
+    if (mWritePosition + size < mWritePosition)
+    {
+        return false;
+    }
+
     if (!reserveSpace(mReservedSpace))
     {
-        return -1;
+        return false;
     }
     mReservedSpace = 0;
 
     if (!mVertexBuffer->storeRawData(data, size, mWritePosition))
     {
-        return -1;
+        return false;
+    }
+
+    if (outStreamOffset)
+    {
+        *outStreamOffset = mWritePosition;
     }
 
-    int oldWritePos = static_cast<int>(mWritePosition);
     mWritePosition += size;
 
-    return oldWritePos;
+    return true;
 }
 
 bool VertexBufferInterface::reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances)
 {
-    unsigned int requiredSpace = mVertexBuffer->getSpaceRequired(attribute, count, instances);
+    unsigned int requiredSpace;
+    if (!mVertexBuffer->getSpaceRequired(attribute, count, instances, &requiredSpace))
+    {
+        return false;
+    }
 
     // Protect against integer overflow
     if (mReservedSpace + requiredSpace < mReservedSpace)
@@ -195,7 +224,7 @@ StaticVertexBufferInterface::~StaticVertexBufferInterface()
 {
 }
 
-int StaticVertexBufferInterface::lookupAttribute(const gl::VertexAttribute &attribute)
+bool StaticVertexBufferInterface::lookupAttribute(const gl::VertexAttribute &attribute, unsigned int *outStreamOffset)
 {
     for (unsigned int element = 0; element < mCache.size(); element++)
     {
@@ -206,12 +235,16 @@ int StaticVertexBufferInterface::lookupAttribute(const gl::VertexAttribute &attr
         {
             if (mCache[element].attributeOffset == attribute.mOffset % attribute.stride())
             {
-                return mCache[element].streamOffset;
+                if (outStreamOffset)
+                {
+                    *outStreamOffset = mCache[element].streamOffset;
+                }
+                return true;
             }
         }
     }
 
-    return -1;
+    return false;
 }
 
 bool StaticVertexBufferInterface::reserveSpace(unsigned int size)
@@ -233,13 +266,27 @@ bool StaticVertexBufferInterface::reserveSpace(unsigned int size)
     }
 }
 
-int StaticVertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances)
+bool StaticVertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances,
+                                                        unsigned int *outStreamOffset)
 {
-    int attributeOffset = attrib.mOffset % attrib.stride();
-    VertexElement element = { attrib.mType, attrib.mSize, attrib.stride(), attrib.mNormalized, attributeOffset, getWritePosition() };
-    mCache.push_back(element);
+    unsigned int streamOffset;
+    if (VertexBufferInterface::storeVertexAttributes(attrib, start, count, instances, &streamOffset))
+    {
+        int attributeOffset = attrib.mOffset % attrib.stride();
+        VertexElement element = { attrib.mType, attrib.mSize, attrib.stride(), attrib.mNormalized, attributeOffset, streamOffset };
+        mCache.push_back(element);
 
-    return VertexBufferInterface::storeVertexAttributes(attrib, start, count, instances);
+        if (outStreamOffset)
+        {
+            *outStreamOffset = streamOffset;
+        }
+
+        return true;
+    }
+    else
+    {
+        return false;
+    }
 }
 
 }

src/libGLESv2/renderer/VertexBuffer.h

@@ -33,8 +33,8 @@ class VertexBuffer
                                        GLsizei instances, unsigned int offset) = 0;
     virtual bool storeRawData(const void* data, unsigned int size, unsigned int offset) = 0;
 
-    virtual unsigned int getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count,
-                                          GLsizei instances) const = 0;
+    virtual bool getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances,
+                                  unsigned int *outSpaceRequired) const = 0;
 
     virtual bool requiresConversion(const gl::VertexAttribute &attrib) const = 0;
 
@@ -67,8 +67,9 @@ class VertexBufferInterface
 
     unsigned int getSerial() const;
 
-    virtual int storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances);
-    virtual int storeRawData(const void* data, unsigned int size);
+    virtual bool storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances,
+                                      unsigned int *outStreamOffset);
+    virtual bool storeRawData(const void* data, unsigned int size, unsigned int *outStreamOffset);
 
     VertexBuffer* getVertexBuffer() const;
 
@@ -110,10 +111,10 @@ class StaticVertexBufferInterface : public VertexBufferInterface
     explicit StaticVertexBufferInterface(rx::Renderer *renderer);
     ~StaticVertexBufferInterface();
 
-    int storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances);
+    bool storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances,
+                               unsigned int *outStreamOffset);
 
-    // Returns the offset into the vertex buffer, or -1 if not found
-    int lookupAttribute(const gl::VertexAttribute &attribute);
+    bool lookupAttribute(const gl::VertexAttribute &attribute, unsigned int* outStreamOffset);
 
   protected:
     bool reserveSpace(unsigned int size);

src/libGLESv2/renderer/VertexBuffer11.cpp

@@ -152,18 +152,40 @@ bool VertexBuffer11::storeRawData(const void* data, unsigned int size, unsigned
     }
 }
 
-unsigned int VertexBuffer11::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count,
-                                              GLsizei instances) const
+bool VertexBuffer11::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count,
+                                      GLsizei instances, unsigned int *outSpaceRequired) const
 {
     unsigned int elementSize = getVertexConversion(attrib).outputElementSize;
 
+    unsigned int elementCount = 0;
     if (instances == 0 || attrib.mDivisor == 0)
     {
-        return elementSize * count;
+        elementCount = count;
     }
     else
     {
-        return elementSize * ((instances + attrib.mDivisor - 1) / attrib.mDivisor);
+        if (static_cast<unsigned int>(instances) < std::numeric_limits<unsigned int>::max() - (attrib.mDivisor - 1))
+        {
+            // Round up
+            elementCount = (static_cast<unsigned int>(instances) + (attrib.mDivisor - 1)) / attrib.mDivisor;
+        }
+        else
+        {
+            elementCount = instances / attrib.mDivisor;
+        }
+    }
+
+    if (elementSize <= std::numeric_limits<unsigned int>::max() / elementCount)
+    {
+        if (outSpaceRequired)
+        {
+            *outSpaceRequired = elementSize * elementCount;
+        }
+        return true;
+    }
+    else
+    {
+        return false;
     }
 }
 

src/libGLESv2/renderer/VertexBuffer11.h

@@ -26,10 +26,11 @@ class VertexBuffer11 : public VertexBuffer
     static VertexBuffer11 *makeVertexBuffer11(VertexBuffer *vetexBuffer);
 
     virtual bool storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances,
-                                 unsigned int offset);
+                                       unsigned int offset);
     virtual bool storeRawData(const void* data, unsigned int size, unsigned int offset);
 
-    virtual unsigned int getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances) const;
+    virtual bool getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances,
+                                  unsigned int *outSpaceRequired) const;
 
     virtual bool requiresConversion(const gl::VertexAttribute &attrib) const;
 
@@ -70,4 +71,4 @@ class VertexBuffer11 : public VertexBuffer
 
 }
 
-#endif // LIBGLESV2_RENDERER_VERTEXBUFFER11_H_
+#endif // LIBGLESV2_RENDERER_VERTEXBUFFER11_H_
\ No newline at end of file

src/libGLESv2/renderer/VertexBuffer9.cpp

@@ -95,7 +95,14 @@ bool VertexBuffer9::storeVertexAttributes(const gl::VertexAttribute &attrib, GLi
         DWORD lockFlags = mDynamicUsage ? D3DLOCK_NOOVERWRITE : 0;
 
         void *mapPtr = NULL;
-        HRESULT result = mVertexBuffer->Lock(offset, spaceRequired(attrib, count, instances), &mapPtr, lockFlags);
+
+        unsigned int mapSize;
+        if (!spaceRequired(attrib, count, instances, &mapSize))
+        {
+            return false;
+        }
+
+        HRESULT result = mVertexBuffer->Lock(offset, mapSize, &mapPtr, lockFlags);
 
         if (FAILED(result))
         {
@@ -167,9 +174,10 @@ bool VertexBuffer9::storeRawData(const void* data, unsigned int size, unsigned i
     }
 }
 
-unsigned int VertexBuffer9::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances) const
+bool VertexBuffer9::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances,
+                                     unsigned int *outSpaceRequired) const
 {
-    return spaceRequired(attrib, count, instances);
+    return spaceRequired(attrib, count, instances, outSpaceRequired);
 }
 
 bool VertexBuffer9::requiresConversion(const gl::VertexAttribute &attrib) const
@@ -179,7 +187,8 @@ bool VertexBuffer9::requiresConversion(const gl::VertexAttribute &attrib) const
 
 unsigned int VertexBuffer9::getVertexSize(const gl::VertexAttribute &attrib) const
 {
-    return spaceRequired(attrib, 1, 0);
+    unsigned int spaceRequired;
+    return getSpaceRequired(attrib, 1, 0, &spaceRequired) ? spaceRequired : 0;
 }
 
 D3DDECLTYPE VertexBuffer9::getDeclType(const gl::VertexAttribute &attrib) const
@@ -469,17 +478,52 @@ const VertexBuffer9::FormatConverter &VertexBuffer9::formatConverter(const gl::V
     return mFormatConverters[typeIndex(attribute.mType)][attribute.mNormalized][attribute.mSize - 1];
 }
 
-unsigned int VertexBuffer9::spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances)
+bool VertexBuffer9::spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances,
+                                  unsigned int *outSpaceRequired)
 {
     unsigned int elementSize = formatConverter(attrib).outputElementSize;
 
     if (instances == 0 || attrib.mDivisor == 0)
     {
-        return elementSize * count;
+        unsigned int elementCount = 0;
+        if (instances == 0 || attrib.mDivisor == 0)
+        {
+            elementCount = count;
+        }
+        else
+        {
+            if (static_cast<unsigned int>(instances) < std::numeric_limits<unsigned int>::max() - (attrib.mDivisor - 1))
+            {
+                // Round up
+                elementCount = (static_cast<unsigned int>(instances) + (attrib.mDivisor - 1)) / attrib.mDivisor;
+            }
+            else
+            {
+                elementCount = static_cast<unsigned int>(instances) / attrib.mDivisor;
+            }
+        }
+
+        if (elementSize <= std::numeric_limits<unsigned int>::max() / elementCount)
+        {
+            if (outSpaceRequired)
+            {
+                *outSpaceRequired = elementSize * elementCount;
+            }
+            return true;
+        }
+        else
+        {
+            return false;
+        }
     }
     else
     {
-        return elementSize * ((instances + attrib.mDivisor - 1) / attrib.mDivisor);
+        const unsigned int elementSize = 4;
+        if (outSpaceRequired)
+        {
+            *outSpaceRequired = elementSize * 4;
+        }
+        return true;
     }
 }
 

src/libGLESv2/renderer/VertexBuffer9.h

@@ -29,7 +29,7 @@ class VertexBuffer9 : public VertexBuffer
                                        unsigned int offset);
     virtual bool storeRawData(const void* data, unsigned int size, unsigned int offset);
 
-    virtual unsigned int getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances) const;
+    virtual bool getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances, unsigned int *outSpaceRequired) const;
 
     virtual bool requiresConversion(const gl::VertexAttribute &attrib) const;
 
@@ -82,7 +82,8 @@ class VertexBuffer9 : public VertexBuffer
     static unsigned int typeIndex(GLenum type);
     static const FormatConverter &formatConverter(const gl::VertexAttribute &attribute);
 
-    static unsigned int spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances);
+    static bool spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances,
+                              unsigned int *outSpaceRequired);
 };
 
 }

src/libGLESv2/renderer/VertexDataManager.cpp

@@ -26,9 +26,15 @@ namespace
 namespace rx
 {
 
-static int elementsInBuffer(const gl::VertexAttribute &attribute, int size)
+static int elementsInBuffer(const gl::VertexAttribute &attribute, unsigned int size)
 {
-    int stride = attribute.stride();
+    // Size cannot be larger than a GLsizei
+    if (size > static_cast<unsigned int>(std::numeric_limits<int>::max()))
+    {
+        size = static_cast<unsigned int>(std::numeric_limits<int>::max());
+    }
+
+    GLsizei stride = attribute.stride();
     return (size - attribute.mOffset % stride + (stride - attribute.typeSize())) / stride;
 }
 
@@ -92,7 +98,7 @@ GLenum VertexDataManager::prepareVertexData(const gl::VertexAttribute attribs[],
             gl::Buffer *buffer = attribs[i].mBoundBuffer.get();
             StaticVertexBufferInterface *staticBuffer = buffer ? buffer->getStaticVertexBuffer() : NULL;
 
-            if (staticBuffer && staticBuffer->getBufferSize() > 0 && staticBuffer->lookupAttribute(attribs[i]) == -1 &&
+            if (staticBuffer && staticBuffer->getBufferSize() > 0 && !staticBuffer->lookupAttribute(attribs[i], NULL) &&
                 !directStoragePossible(staticBuffer, attribs[i]))
             {
                 buffer->invalidateStaticData();
@@ -155,7 +161,7 @@ GLenum VertexDataManager::prepareVertexData(const gl::VertexAttribute attribs[],
                 BufferStorage *storage = buffer ? buffer->getStorage() : NULL;
                 bool directStorage = directStoragePossible(vertexBuffer, attribs[i]);
 
-                std::size_t streamOffset = -1;
+                unsigned int streamOffset = 0;
                 unsigned int outputElementSize = 0;
 
                 if (directStorage)
@@ -166,37 +172,39 @@ GLenum VertexDataManager::prepareVertexData(const gl::VertexAttribute attribs[],
                 }
                 else if (staticBuffer)
                 {
-                    streamOffset = staticBuffer->lookupAttribute(attribs[i]);
-                    outputElementSize = staticBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0);
+                    if (!staticBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0, &outputElementSize))
+                    {
+                        return GL_OUT_OF_MEMORY;
+                    }
 
-                    if (streamOffset == -1)
+                    if (!staticBuffer->lookupAttribute(attribs[i], &streamOffset))
                     {
                         // Convert the entire buffer
                         int totalCount = elementsInBuffer(attribs[i], storage->getSize());
                         int startIndex = attribs[i].mOffset / attribs[i].stride();
 
-                        streamOffset = staticBuffer->storeVertexAttributes(attribs[i], -startIndex, totalCount, 0);
+                        if (!staticBuffer->storeVertexAttributes(attribs[i], -startIndex, totalCount, 0, &streamOffset))
+                        {
+                            return GL_OUT_OF_MEMORY;
+                        }
                     }
 
-                    if (streamOffset != -1)
+                    unsigned int firstElementOffset = (attribs[i].mOffset / attribs[i].stride()) * outputElementSize;
+                    unsigned int startOffset = (instances == 0 || attribs[i].mDivisor == 0) ? start * outputElementSize : 0;
+                    if (streamOffset + firstElementOffset + startOffset < streamOffset)
                     {
-                        streamOffset += (attribs[i].mOffset / attribs[i].stride()) * outputElementSize;
-
-                        if (instances == 0 || attribs[i].mDivisor == 0)
-                        {
-                            streamOffset += start * outputElementSize;
-                        }
+                        return GL_OUT_OF_MEMORY;
                     }
+
+                    streamOffset += firstElementOffset + startOffset;
                 }
                 else
                 {
-                    outputElementSize = mStreamingBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0);
-                    streamOffset = mStreamingBuffer->storeVertexAttributes(attribs[i], start, count, instances);
-                }
-
-                if (streamOffset == -1)
-                {
-                    return GL_OUT_OF_MEMORY;
+                    if (!mStreamingBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0, &outputElementSize) ||
+                        !mStreamingBuffer->storeVertexAttributes(attribs[i], start, count, instances, &streamOffset))
+                    {
+                        return GL_OUT_OF_MEMORY;
+                    }
                 }
 
                 translated[i].storage = directStorage ? storage : NULL;
@@ -227,8 +235,9 @@ GLenum VertexDataManager::prepareVertexData(const gl::VertexAttribute attribs[],
                     {
                         return GL_OUT_OF_MEMORY;
                     }
-                    int streamOffset = buffer->storeRawData(attribs[i].mCurrentValue, requiredSpace);
-                    if (streamOffset == -1)
+
+                    unsigned int streamOffset;
+                    if (!buffer->storeRawData(attribs[i].mCurrentValue, requiredSpace, &streamOffset))
                     {
                         return GL_OUT_OF_MEMORY;
                     }

src/libGLESv2/renderer/VertexDataManager.h

@@ -31,8 +31,8 @@ struct TranslatedAttribute
     bool active;
 
     const gl::VertexAttribute *attribute;
-    UINT offset;
-    UINT stride;   // 0 means not to advance the read pointer at all
+    unsigned int offset;
+    unsigned int stride;   // 0 means not to advance the read pointer at all
 
     VertexBuffer *vertexBuffer;
     BufferStorage *storage;