Handle buffer memory allocation failures gracefully.

src/libGLESv2/renderer/d3d11/BufferStorage11.cpp

@@ -87,7 +87,7 @@ class BufferStorage11::TypedBufferStorage11
 
     virtual bool copyFromStorage(TypedBufferStorage11 *source, size_t sourceOffset,
                                  size_t size, size_t destOffset) = 0;
-    virtual void resize(size_t size, bool preserveData) = 0;
+    virtual bool resize(size_t size, bool preserveData) = 0;
 
     virtual void *map(GLbitfield access) = 0;
     virtual void unmap() = 0;
@@ -113,7 +113,7 @@ class BufferStorage11::NativeBuffer11 : public BufferStorage11::TypedBufferStora
 
     virtual bool copyFromStorage(TypedBufferStorage11 *source, size_t sourceOffset,
                                  size_t size, size_t destOffset);
-    virtual void resize(size_t size, bool preserveData);
+    virtual bool resize(size_t size, bool preserveData);
 
     virtual void *map(GLbitfield access);
     virtual void unmap();
@@ -134,7 +134,7 @@ class BufferStorage11::PackStorage11 : public BufferStorage11::TypedBufferStorag
 
     virtual bool copyFromStorage(TypedBufferStorage11 *source, size_t sourceOffset,
                                  size_t size, size_t destOffset);
-    virtual void resize(size_t size, bool preserveData);
+    virtual bool resize(size_t size, bool preserveData);
 
     virtual void *map(GLbitfield access);
     virtual void unmap();
@@ -148,8 +148,9 @@ class BufferStorage11::PackStorage11 : public BufferStorage11::TypedBufferStorag
     ID3D11Texture2D *mStagingTexture;
     DXGI_FORMAT mTextureFormat;
     gl::Extents mTextureSize;
-    unsigned char *mMemoryBuffer;
+    std::vector<unsigned char> mMemoryBuffer;
     PackPixelsParams *mQueuedPackCommand;
+    PackPixelsParams mPackParams;
     bool mDataModified;
 };
 
@@ -179,6 +180,13 @@ BufferStorage11 *BufferStorage11::makeBufferStorage11(BufferStorage *bufferStora
 void *BufferStorage11::getData()
 {
     NativeBuffer11 *stagingBuffer = getStagingBuffer();
+
+    if (!stagingBuffer)
+    {
+        // Out-of-memory
+        return NULL;
+    }
+
     if (stagingBuffer->getDataRevision() > mResolvedDataRevision)
     {
         if (stagingBuffer->getSize() > mResolvedData.size())
@@ -216,12 +224,22 @@ void BufferStorage11::setData(const void* data, size_t size, size_t offset)
     {
         NativeBuffer11 *stagingBuffer = getStagingBuffer();
 
+        if (!stagingBuffer)
+        {
+            // Out-of-memory
+            return;
+        }
+
         // Explicitly resize the staging buffer, preserving data if the new data will not
         // completely fill the buffer
         if (stagingBuffer->getSize() < requiredSize)
         {
             bool preserveData = (offset > 0);
-            stagingBuffer->resize(requiredSize, preserveData);
+            if (!stagingBuffer->resize(requiredSize, preserveData))
+            {
+                // Out-of-memory
+                return;
+            }
         }
 
         ID3D11DeviceContext *context = mRenderer->getDeviceContext();
@@ -273,7 +291,11 @@ void BufferStorage11::clear()
 void BufferStorage11::markTransformFeedbackUsage()
 {
     TypedBufferStorage11 *transformFeedbackStorage = getStorage(BUFFER_USAGE_VERTEX_OR_TRANSFORM_FEEDBACK);
-    transformFeedbackStorage->setDataRevision(transformFeedbackStorage->getDataRevision() + 1);
+
+    if (transformFeedbackStorage)
+    {
+        transformFeedbackStorage->setDataRevision(transformFeedbackStorage->getDataRevision() + 1);
+    }
 }
 
 size_t BufferStorage11::getSize() const
@@ -304,6 +326,13 @@ ID3D11Buffer *BufferStorage11::getBuffer(BufferUsage usage)
     markBufferUsage();
 
     TypedBufferStorage11 *typedBuffer = getStorage(usage);
+
+    if (!typedBuffer)
+    {
+        // Storage out-of-memory
+        return NULL;
+    }
+
     ASSERT(HAS_DYNAMIC_TYPE(NativeBuffer11*, typedBuffer));
 
     return static_cast<NativeBuffer11*>(typedBuffer)->getNativeBuffer();
@@ -312,6 +341,13 @@ ID3D11Buffer *BufferStorage11::getBuffer(BufferUsage usage)
 ID3D11ShaderResourceView *BufferStorage11::getSRV(DXGI_FORMAT srvFormat)
 {
     TypedBufferStorage11 *storage = getStorage(BUFFER_USAGE_PIXEL_UNPACK);
+
+    if (!storage)
+    {
+        // Storage out-of-memory
+        return NULL;
+    }
+
     ASSERT(HAS_DYNAMIC_TYPE(NativeBuffer11*, storage));
     ID3D11Buffer *buffer = static_cast<NativeBuffer11*>(storage)->getNativeBuffer();
 
@@ -353,8 +389,11 @@ void BufferStorage11::packPixels(ID3D11Texture2D *srcTexture, UINT srcSubresourc
 
     TypedBufferStorage11 *latestStorage = getLatestStorage();
 
-    packStorage->packPixels(srcTexture, srcSubresource, params);
-    packStorage->setDataRevision(latestStorage ? latestStorage->getDataRevision() + 1 : 1);
+    if (packStorage && latestStorage)
+    {
+        packStorage->packPixels(srcTexture, srcSubresource, params);
+        packStorage->setDataRevision(latestStorage ? latestStorage->getDataRevision() + 1 : 1);
+    }
 }
 
 BufferStorage11::TypedBufferStorage11 *BufferStorage11::getStorage(BufferUsage usage)
@@ -384,7 +423,11 @@ BufferStorage11::TypedBufferStorage11 *BufferStorage11::getStorage(BufferUsage u
     // resize buffer
     if (directBuffer->getSize() < mSize)
     {
-        directBuffer->resize(mSize, true);
+        if (!directBuffer->resize(mSize, true))
+        {
+            // Out of memory error
+            return NULL;
+        }
     }
 
     TypedBufferStorage11 *latestBuffer = getLatestStorage();
@@ -442,6 +485,12 @@ void *BufferStorage11::map(GLbitfield access)
         mMappedStorage = getStagingBuffer();
     }
 
+    if (!mMappedStorage)
+    {
+        // Out-of-memory
+        return NULL;
+    }
+
     return mMappedStorage->map(access);
 }
 
@@ -455,6 +504,13 @@ void BufferStorage11::unmap()
 BufferStorage11::NativeBuffer11 *BufferStorage11::getStagingBuffer()
 {
     TypedBufferStorage11 *stagingStorage = getStorage(BUFFER_USAGE_STAGING);
+
+    if (!stagingStorage)
+    {
+        // Out-of-memory
+        return NULL;
+    }
+
     ASSERT(HAS_DYNAMIC_TYPE(NativeBuffer11*, stagingStorage));
     return static_cast<NativeBuffer11*>(stagingStorage);
 }
@@ -462,6 +518,13 @@ BufferStorage11::NativeBuffer11 *BufferStorage11::getStagingBuffer()
 BufferStorage11::PackStorage11 *BufferStorage11::getPackStorage()
 {
     TypedBufferStorage11 *packStorage = getStorage(BUFFER_USAGE_PIXEL_PACK);
+
+    if (!packStorage)
+    {
+        // Out-of-memory
+        return NULL;
+    }
+
     ASSERT(HAS_DYNAMIC_TYPE(PackStorage11*, packStorage));
     return static_cast<PackStorage11*>(packStorage);
 }
@@ -525,7 +588,7 @@ bool BufferStorage11::NativeBuffer11::copyFromStorage(TypedBufferStorage11 *sour
     return createBuffer;
 }
 
-void BufferStorage11::NativeBuffer11::resize(size_t size, bool preserveData)
+bool BufferStorage11::NativeBuffer11::resize(size_t size, bool preserveData)
 {
     ID3D11Device *device = mRenderer->getDevice();
     ID3D11DeviceContext *context = mRenderer->getDeviceContext();
@@ -538,7 +601,7 @@ void BufferStorage11::NativeBuffer11::resize(size_t size, bool preserveData)
 
     if (FAILED(result))
     {
-        return gl::error(GL_OUT_OF_MEMORY);
+        return gl::error(GL_OUT_OF_MEMORY, false);
     }
 
     if (mNativeBuffer && preserveData)
@@ -559,6 +622,8 @@ void BufferStorage11::NativeBuffer11::resize(size_t size, bool preserveData)
     mNativeBuffer = newBuffer;
 
     mBufferSize = bufferDesc.ByteWidth;
+
+    return true;
 }
 
 void BufferStorage11::NativeBuffer11::fillBufferDesc(D3D11_BUFFER_DESC* bufferDesc, Renderer *renderer,
@@ -636,7 +701,6 @@ BufferStorage11::PackStorage11::PackStorage11(Renderer11 *renderer)
     : TypedBufferStorage11(renderer, BUFFER_USAGE_PIXEL_PACK),
       mStagingTexture(NULL),
       mTextureFormat(DXGI_FORMAT_UNKNOWN),
-      mMemoryBuffer(NULL),
       mQueuedPackCommand(NULL),
       mDataModified(false)
 {
@@ -645,7 +709,6 @@ BufferStorage11::PackStorage11::PackStorage11(Renderer11 *renderer)
 BufferStorage11::PackStorage11::~PackStorage11()
 {
     SafeRelease(mStagingTexture);
-    SafeDeleteArray(mMemoryBuffer);
     SafeDelete(mQueuedPackCommand);
 }
 
@@ -656,23 +719,15 @@ bool BufferStorage11::PackStorage11::copyFromStorage(TypedBufferStorage11 *sourc
     return false;
 }
 
-void BufferStorage11::PackStorage11::resize(size_t size, bool preserveData)
+bool BufferStorage11::PackStorage11::resize(size_t size, bool preserveData)
 {
     if (size != mBufferSize)
     {
-        unsigned char *existingData = mMemoryBuffer;
-
-        // Allocate new memory buffer
-        mMemoryBuffer = new unsigned char[size];
-        memset(mMemoryBuffer, 0, sizeof(unsigned char) * size);
-
-        if (existingData && preserveData)
-        {
-            memcpy(mMemoryBuffer, existingData, std::min(mBufferSize, size));
-        }
-
+        mMemoryBuffer.resize(size, 0);
         mBufferSize = size;
     }
+
+    return true;
 }
 
 void *BufferStorage11::PackStorage11::map(GLbitfield access)
@@ -684,7 +739,8 @@ void *BufferStorage11::PackStorage11::map(GLbitfield access)
 
     flushQueuedPackCommand();
     mDataModified = (mDataModified || (access & GL_MAP_WRITE_BIT) != 0);
-    return mMemoryBuffer;
+
+    return &mMemoryBuffer[0];
 }
 
 void BufferStorage11::PackStorage11::unmap()
@@ -758,11 +814,11 @@ void BufferStorage11::PackStorage11::packPixels(ID3D11Texture2D *srcTexure, UINT
 
 void BufferStorage11::PackStorage11::flushQueuedPackCommand()
 {
-    ASSERT(mMemoryBuffer);
+    ASSERT(!mMemoryBuffer.empty());
 
     if (mQueuedPackCommand)
     {
-        mRenderer->packPixels(mStagingTexture, *mQueuedPackCommand, mMemoryBuffer);
+        mRenderer->packPixels(mStagingTexture, *mQueuedPackCommand, &mMemoryBuffer[0]);
         SafeDelete(mQueuedPackCommand);
     }
 }

src/libGLESv2/renderer/d3d9/BufferStorage9.cpp

@@ -9,6 +9,7 @@
 
 #include "libGLESv2/renderer/d3d9/BufferStorage9.h"
 #include "common/debug.h"
+#include "libGLESv2/main.h"
 
 namespace rx
 {

src/libGLESv2/renderer/d3d9/VertexBuffer9.cpp

@@ -121,7 +121,8 @@ bool VertexBuffer9::storeVertexAttributes(const gl::VertexAttribute &attrib, con
 
         if (!needsConversion && inputStride == elementSize)
         {
-            memcpy(mapPtr, input, count * inputStride);
+            size_t copySize = static_cast<size_t>(count) * static_cast<size_t>(inputStride);
+            memcpy(mapPtr, input, copySize);
         }
         else
         {

tests/angle_tests/BufferDataNULLTest.cpp

-#include "ANGLETest.h"
-
-class BufferDataNULLTest : public ANGLETest
-{
-protected:
-    BufferDataNULLTest()
-    {
-        setWindowWidth(1);
-        setWindowHeight(1);
-    }
-};
-
-TEST_F(BufferDataNULLTest, null_data)
-{
-    GLuint buf;
-    glGenBuffers(1, &buf);
-    ASSERT_NE(buf, 0U);
-
-    glBindBuffer(GL_ARRAY_BUFFER, buf);
-    EXPECT_GL_NO_ERROR();
-
-    const int numIterations = 128;
-    for (int i = 0; i < numIterations; ++i)
-    {
-        GLsizei bufferSize = sizeof(GLfloat) * (i + 1);
-        glBufferData(GL_ARRAY_BUFFER, bufferSize, NULL, GL_STATIC_DRAW);
-        EXPECT_GL_NO_ERROR();
-
-        for (int j = 0; j < bufferSize; j++)
-        {
-            for (int k = 0; k < bufferSize - j; k++)
-            {
-                glBufferSubData(GL_ARRAY_BUFFER, k, j, NULL);
-                EXPECT_GL_NO_ERROR();
-            }
-        }
-    }
-
-    glDeleteBuffers(1, &buf);
-    EXPECT_GL_NO_ERROR();
-}

tests/angle_tests/BufferDataTest.cpp

+#include "ANGLETest.h"
+
+class BufferDataTest : public ANGLETest
+{
+protected:
+    BufferDataTest()
+        : mBuffer(0),
+          mProgram(0),
+          mAttribLocation(-1)
+    {
+        setWindowWidth(16);
+        setWindowHeight(16);
+        setConfigRedBits(8);
+        setConfigGreenBits(8);
+        setConfigBlueBits(8);
+        setConfigAlphaBits(8);
+        setConfigDepthBits(24);
+    }
+
+    virtual void SetUp()
+    {
+        ANGLETest::SetUp();
+
+        const char * vsSource = SHADER_SOURCE
+        (
+            attribute vec4 position;
+            attribute float in_attrib;
+            varying float v_attrib;
+            void main()
+            {
+                v_attrib = in_attrib;
+                gl_Position = position;
+            }
+        );
+
+        const char * fsSource = SHADER_SOURCE
+        (
+            precision mediump float;
+            varying float v_attrib;
+            void main()
+            {
+                gl_FragColor = vec4(v_attrib, 0, 0, 1);
+            }
+        );
+
+        glGenBuffers(1, &mBuffer);
+        ASSERT_NE(mBuffer, 0U);
+
+        mProgram = compileProgram(vsSource, fsSource);
+        ASSERT_NE(mProgram, 0U);
+
+        mAttribLocation = glGetAttribLocation(mProgram, "in_attrib");
+        ASSERT_NE(mAttribLocation, -1);
+
+        glClearColor(0, 0, 0, 0);
+        glClearDepthf(0.0);
+        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+
+        glDisable(GL_DEPTH_TEST);
+
+        ASSERT_GL_NO_ERROR();
+    }
+
+    virtual void TearDown()
+    {
+        glDeleteBuffers(1, &mBuffer);
+        glDeleteProgram(mProgram);
+
+        ANGLETest::TearDown();
+    }
+
+    GLuint mBuffer;
+    GLuint mProgram;
+    GLint mAttribLocation;
+};
+
+TEST_F(BufferDataTest, null_data)
+{
+    glBindBuffer(GL_ARRAY_BUFFER, mBuffer);
+    EXPECT_GL_NO_ERROR();
+
+    const int numIterations = 128;
+    for (int i = 0; i < numIterations; ++i)
+    {
+        GLsizei bufferSize = sizeof(GLfloat) * (i + 1);
+        glBufferData(GL_ARRAY_BUFFER, bufferSize, NULL, GL_STATIC_DRAW);
+        EXPECT_GL_NO_ERROR();
+
+        for (int j = 0; j < bufferSize; j++)
+        {
+            for (int k = 0; k < bufferSize - j; k++)
+            {
+                glBufferSubData(GL_ARRAY_BUFFER, k, j, NULL);
+                EXPECT_GL_NO_ERROR();
+            }
+        }
+    }
+}
+
+TEST_F(BufferDataTest, huge_setdata_should_not_crash)
+{
+    glBindBuffer(GL_ARRAY_BUFFER, mBuffer);
+    EXPECT_GL_NO_ERROR();
+
+    // use as large a size as possible without causing an exception
+    GLsizei hugeSize = (1 << 30);
+
+    // on x64, use as large a GLsizei value as possible
+    if (sizeof(size_t) > 4)
+    {
+        hugeSize = std::numeric_limits<GLsizei>::max();
+    }
+
+    char *data = new (std::nothrow) char[hugeSize];
+    EXPECT_NE((char * const)NULL, data);
+
+    if (data == NULL)
+    {
+        return;
+    }
+
+    memset(data, 0, hugeSize);
+
+    float * fValue = reinterpret_cast<float*>(data);
+    for (unsigned int f = 0; f < 6; f++)
+    {
+        fValue[f] = 1.0f;
+    }
+
+    glBufferData(GL_ARRAY_BUFFER, hugeSize, data, GL_STATIC_DRAW);
+
+    GLenum error = glGetError();
+    if (error == GL_NO_ERROR)
+    {
+        // If we didn't fail because of an out of memory error, try drawing a quad
+        // using the large buffer
+
+        // DISABLED because it takes a long time, but left for posterity
+
+        //glUseProgram(mProgram);
+        //glVertexAttribPointer(mAttribLocation, 1, GL_FLOAT, GL_FALSE, 4, NULL);
+        //glEnableVertexAttribArray(mAttribLocation);
+        //glBindBuffer(GL_ARRAY_BUFFER, 0);
+        //drawQuad(mProgram, "position", 0.5f);
+        //swapBuffers();
+
+        //// Draw operations can also generate out-of-memory, which is in-spec
+        //error = glGetError();
+        //if (error == GL_NO_ERROR)
+        //{
+        //    GLint viewportSize[4];
+        //    glGetIntegerv(GL_VIEWPORT, viewportSize);
+
+        //    GLint midPixelX = (viewportSize[0] + viewportSize[2]) / 2;
+        //    GLint midPixelY = (viewportSize[1] + viewportSize[3]) / 2;
+
+        //    EXPECT_PIXEL_EQ(midPixelX, midPixelY, 255, 0, 0, 255);
+        //}
+        //else
+        //{
+        //    EXPECT_EQ(GL_OUT_OF_MEMORY, error);
+        //}
+    }
+    else
+    {
+        EXPECT_EQ(GL_OUT_OF_MEMORY, error);
+    }
+
+    delete[] data;
+}
+