Fix integer overflow in image size calculation

GLsizei is historically defined as a 32-bit signed integer, which is not allowed to be negative. But doing arithmetic with them can lead to overflows producing undefined values, including negative ones. Use size_t instead for these image data size calculations. It's defined as an unsigned integer large enough to fit valid memory buffer sizes. In practice that's 32-bit on 32-bit platforms, and 64-bit on 64-bit ones, which is large enough to avoid overflow in the OutOfMemory test. Bug: b/140421004 Change-Id: Id05dd64d470f579455fdc9925b0ab303ecaf37fb Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/36968Tested-by: Nicolas Capens <nicolascapens@google.com> Reviewed-by: Alexis Hétu <sugoi@google.com>

Fix integer overflow in image size calculation
5be09849 · Nicolas Capens · Nicolas Capens · 391532fe · 5be09849 · 5be09849
Commit 5be09849 authored Oct 02, 2019 by Nicolas Capens Committed by Nicolas Capens Oct 02, 2019
Hide whitespace changes
Inline Side-by-side

Showing with 7 additions and 6 deletions

Context.cpp src/OpenGL/libGLESv2/Context.cpp +5 -4

Context.h src/OpenGL/libGLESv2/Context.h +2 -2

No files found.
--- a/src/OpenGL/libGLESv2/Context.cpp
+++ b/src/OpenGL/libGLESv2/Context.cpp
@@ -1567,15 +1567,16 @@ Buffer *Context::getGenericUniformBuffer() const
 	return mState.genericUniformBuffer;
 }

-GLsizei Context::getRequiredBufferSize(GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type) const
+size_t Context::getRequiredBufferSize(GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type) const
 {
 	GLsizei inputWidth = (mState.unpackParameters.rowLength == 0) ? width : mState.unpackParameters.rowLength;
 	GLsizei inputPitch = gl::ComputePitch(inputWidth, format, type, mState.unpackParameters.alignment);
 	GLsizei inputHeight = (mState.unpackParameters.imageHeight == 0) ? height : mState.unpackParameters.imageHeight;
-	return inputPitch * inputHeight * depth;
+
+	return static_cast<size_t>(inputPitch) * inputHeight * depth;
 }

-GLenum Context::getPixels(const GLvoid **pixels, GLenum type, GLsizei imageSize) const
+GLenum Context::getPixels(const GLvoid **pixels, GLenum type, size_t imageSize) const
 {
 	if(mState.pixelUnpackBuffer)
 	{
@@ -1598,7 +1599,7 @@ GLenum Context::getPixels(const GLvoid **pixels, GLenum type, GLsizei imageSize)
 			return GL_INVALID_OPERATION;
 		}

-		if(mState.pixelUnpackBuffer->size() - offset < static_cast<size_t>(imageSize))
+		if(mState.pixelUnpackBuffer->size() - offset < imageSize)
 		{
 			return GL_INVALID_OPERATION;
 		}

--- a/src/OpenGL/libGLESv2/Context.h
+++ b/src/OpenGL/libGLESv2/Context.h
@@ -637,8 +637,8 @@ public:
 	Buffer *getPixelPackBuffer() const;
 	Buffer *getPixelUnpackBuffer() const;
 	Buffer *getGenericUniformBuffer() const;
-	GLsizei getRequiredBufferSize(GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type) const;
-	GLenum getPixels(const GLvoid **data, GLenum type, GLsizei imageSize) const;
+	size_t getRequiredBufferSize(GLsizei width, GLsizei height, GLsizei depth, GLenum format, GLenum type) const;
+	GLenum getPixels(const GLvoid **data, GLenum type, size_t imageSize) const;
 	bool getBuffer(GLenum target, es2::Buffer **buffer) const;
 	Program *getCurrentProgram() const;
 	Texture *getTargetTexture(GLenum target) const;