Validate allocation size in PoolAllocator

src/OpenGL/compiler/PoolAlloc.cpp

@@ -227,24 +227,27 @@ void TPoolAllocator::popAll()
 
 void* TPoolAllocator::allocate(size_t numBytes)
 {
+    //
+    // Just keep some interesting statistics.
+    //
+    ++numCalls;
+    totalBytes += numBytes;
+
     // If we are using guard blocks, all allocations are bracketed by
     // them: [guardblock][allocation][guardblock].  numBytes is how
     // much memory the caller asked for.  allocationSize is the total
     // size including guard blocks.  In release build,
     // guardBlockSize=0 and this all gets optimized away.
     size_t allocationSize = TAllocation::allocationSize(numBytes);
-    
-    //
-    // Just keep some interesting statistics.
-    //
-    ++numCalls;
-    totalBytes += numBytes;
+    // Detect integer overflow.
+    if (allocationSize < numBytes)
+        return 0;
 
     //
     // Do the allocation, most likely case first, for efficiency.
     // This step could be moved to be inline sometime.
     //
-    if (currentPageOffset + allocationSize <= pageSize) {
+    if (allocationSize <= pageSize - currentPageOffset) {
         //
         // Safe to allocate from currentPageOffset.
         //
@@ -255,12 +258,16 @@ void* TPoolAllocator::allocate(size_t numBytes)
         return initializeAllocation(inUseList, memory, numBytes);
     }
 
-    if (allocationSize + headerSkip > pageSize) {
+    if (allocationSize > pageSize - headerSkip) {
         //
         // Do a multi-page allocation.  Don't mix these with the others.
         // The OS is efficient and allocating and free-ing multiple pages.
         //
         size_t numBytesToAlloc = allocationSize + headerSkip;
+        // Detect integer overflow.
+        if (numBytesToAlloc < allocationSize)
+            return 0;
+
         tHeader* memory = reinterpret_cast<tHeader*>(::new char[numBytesToAlloc]);
         if (memory == 0)
             return 0;

src/Radiance/compiler/PoolAlloc.cpp

@@ -227,24 +227,27 @@ void TPoolAllocator::popAll()
 
 void* TPoolAllocator::allocate(size_t numBytes)
 {
+    //
+    // Just keep some interesting statistics.
+    //
+    ++numCalls;
+    totalBytes += numBytes;
+
     // If we are using guard blocks, all allocations are bracketed by
     // them: [guardblock][allocation][guardblock].  numBytes is how
     // much memory the caller asked for.  allocationSize is the total
     // size including guard blocks.  In release build,
     // guardBlockSize=0 and this all gets optimized away.
     size_t allocationSize = TAllocation::allocationSize(numBytes);
-    
-    //
-    // Just keep some interesting statistics.
-    //
-    ++numCalls;
-    totalBytes += numBytes;
+    // Detect integer overflow.
+    if (allocationSize < numBytes)
+        return 0;
 
     //
     // Do the allocation, most likely case first, for efficiency.
     // This step could be moved to be inline sometime.
     //
-    if (currentPageOffset + allocationSize <= pageSize) {
+    if (allocationSize <= pageSize - currentPageOffset) {
         //
         // Safe to allocate from currentPageOffset.
         //
@@ -255,12 +258,16 @@ void* TPoolAllocator::allocate(size_t numBytes)
         return initializeAllocation(inUseList, memory, numBytes);
     }
 
-    if (allocationSize + headerSkip > pageSize) {
+    if (allocationSize > pageSize - headerSkip) {
         //
         // Do a multi-page allocation.  Don't mix these with the others.
         // The OS is efficient and allocating and free-ing multiple pages.
         //
         size_t numBytesToAlloc = allocationSize + headerSkip;
+        // Detect integer overflow.
+        if (numBytesToAlloc < allocationSize)
+            return 0;
+
         tHeader* memory = reinterpret_cast<tHeader*>(::new char[numBytesToAlloc]);
         if (memory == 0)
             return 0;