Fix handling integer overflow in constant folding

src/compiler/translator/ConstantUnion.cpp

@@ -61,6 +61,38 @@ T CheckedMul(base::CheckedNumeric<T> lhs,
     return result.ValueOrDefault(0);
 }
 
+// Unsigned types are defined to do arithmetic modulo 2^n in C++. For signed types, overflow
+// behavior is undefined.
+
+template <typename T>
+T WrappingSum(T lhs, T rhs)
+{
+    uint32_t lhsUnsigned = static_cast<uint32_t>(lhs);
+    uint32_t rhsUnsigned = static_cast<uint32_t>(rhs);
+    return static_cast<T>(lhsUnsigned + rhsUnsigned);
+}
+
+template <typename T>
+T WrappingDiff(T lhs, T rhs)
+{
+    uint32_t lhsUnsigned = static_cast<uint32_t>(lhs);
+    uint32_t rhsUnsigned = static_cast<uint32_t>(rhs);
+    return static_cast<T>(lhsUnsigned - rhsUnsigned);
+}
+
+int32_t WrappingMul(int32_t lhs, int32_t rhs)
+{
+    int64_t lhsWide = static_cast<int64_t>(lhs);
+    int64_t rhsWide = static_cast<int64_t>(rhs);
+    // The multiplication is guaranteed not to overflow.
+    int64_t resultWide = lhsWide * rhsWide;
+    // Implement the desired wrapping behavior by masking out the high-order 32 bits.
+    resultWide = resultWide & 0xffffffffll;
+    // Casting to a narrower signed type is fine since the casted value is representable in the
+    // narrower type.
+    return static_cast<int32_t>(resultWide);
+}
+
 }  // anonymous namespace
 
 TConstantUnion::TConstantUnion()
@@ -283,10 +315,10 @@ TConstantUnion TConstantUnion::add(const TConstantUnion &lhs,
     switch (lhs.type)
     {
         case EbtInt:
-            returnValue.setIConst(CheckedSum<int>(lhs.iConst, rhs.iConst, diag, line));
+            returnValue.setIConst(WrappingSum<int>(lhs.iConst, rhs.iConst));
             break;
         case EbtUInt:
-            returnValue.setUConst(CheckedSum<unsigned int>(lhs.uConst, rhs.uConst, diag, line));
+            returnValue.setUConst(WrappingSum<unsigned int>(lhs.uConst, rhs.uConst));
             break;
         case EbtFloat:
             returnValue.setFConst(CheckedSum<float>(lhs.fConst, rhs.fConst, diag, line));
@@ -309,10 +341,10 @@ TConstantUnion TConstantUnion::sub(const TConstantUnion &lhs,
     switch (lhs.type)
     {
         case EbtInt:
-            returnValue.setIConst(CheckedDiff<int>(lhs.iConst, rhs.iConst, diag, line));
+            returnValue.setIConst(WrappingDiff<int>(lhs.iConst, rhs.iConst));
             break;
         case EbtUInt:
-            returnValue.setUConst(CheckedDiff<unsigned int>(lhs.uConst, rhs.uConst, diag, line));
+            returnValue.setUConst(WrappingDiff<unsigned int>(lhs.uConst, rhs.uConst));
             break;
         case EbtFloat:
             returnValue.setFConst(CheckedDiff<float>(lhs.fConst, rhs.fConst, diag, line));
@@ -335,10 +367,12 @@ TConstantUnion TConstantUnion::mul(const TConstantUnion &lhs,
     switch (lhs.type)
     {
         case EbtInt:
-            returnValue.setIConst(CheckedMul<int>(lhs.iConst, rhs.iConst, diag, line));
+            returnValue.setIConst(WrappingMul(lhs.iConst, rhs.iConst));
             break;
         case EbtUInt:
-            returnValue.setUConst(CheckedMul<unsigned int>(lhs.uConst, rhs.uConst, diag, line));
+            // Unsigned integer math in C++ is defined to be done in modulo 2^n, so we rely on that
+            // to implement wrapping multiplication.
+            returnValue.setUConst(lhs.uConst * rhs.uConst);
             break;
         case EbtFloat:
             returnValue.setFConst(CheckedMul<float>(lhs.fConst, rhs.fConst, diag, line));

src/tests/compiler_tests/ConstantFolding_test.cpp

@@ -843,3 +843,123 @@ TEST_F(ConstantFoldingTest, FoldDivideMinimumIntegerByMinusOne)
     compile(shaderString);
     ASSERT_TRUE(constantFoundInAST(0x7fffffff) || constantFoundInAST(-0x7fffffff - 1));
 }
+
+// Test that folding an unsigned integer addition that overflows works.
+// ESSL 3.00.6 section 4.1.3 Integers:
+// "For all precisions, operations resulting in overflow or underflow will not cause any exception,
+// nor will they saturate, rather they will 'wrap' to yield the low-order n bits of the result where
+// n is the size in bits of the integer."
+TEST_F(ConstantFoldingTest, FoldUnsignedIntegerAddOverflow)
+{
+    const std::string &shaderString =
+        "#version 300 es\n"
+        "precision mediump float;\n"
+        "out vec4 my_FragColor;\n"
+        "void main()\n"
+        "{\n"
+        "    uint u = 0xffffffffu + 43u;\n"
+        "    my_FragColor = vec4(u);\n"
+        "}\n";
+    compile(shaderString);
+    ASSERT_TRUE(constantFoundInAST(42u));
+}
+
+// Test that folding a signed integer addition that overflows works.
+// ESSL 3.00.6 section 4.1.3 Integers:
+// "For all precisions, operations resulting in overflow or underflow will not cause any exception,
+// nor will they saturate, rather they will 'wrap' to yield the low-order n bits of the result where
+// n is the size in bits of the integer."
+TEST_F(ConstantFoldingTest, FoldSignedIntegerAddOverflow)
+{
+    const std::string &shaderString =
+        "#version 300 es\n"
+        "precision mediump float;\n"
+        "out vec4 my_FragColor;\n"
+        "void main()\n"
+        "{\n"
+        "    int i = 0x7fffffff + 4;\n"
+        "    my_FragColor = vec4(i);\n"
+        "}\n";
+    compile(shaderString);
+    ASSERT_TRUE(constantFoundInAST(-0x7ffffffd));
+}
+
+// Test that folding an unsigned integer subtraction that overflows works.
+// ESSL 3.00.6 section 4.1.3 Integers:
+// "For all precisions, operations resulting in overflow or underflow will not cause any exception,
+// nor will they saturate, rather they will 'wrap' to yield the low-order n bits of the result where
+// n is the size in bits of the integer."
+TEST_F(ConstantFoldingTest, FoldUnsignedIntegerDiffOverflow)
+{
+    const std::string &shaderString =
+        "#version 300 es\n"
+        "precision mediump float;\n"
+        "out vec4 my_FragColor;\n"
+        "void main()\n"
+        "{\n"
+        "    uint u = 0u - 5u;\n"
+        "    my_FragColor = vec4(u);\n"
+        "}\n";
+    compile(shaderString);
+    ASSERT_TRUE(constantFoundInAST(0xfffffffbu));
+}
+
+// Test that folding a signed integer subtraction that overflows works.
+// ESSL 3.00.6 section 4.1.3 Integers:
+// "For all precisions, operations resulting in overflow or underflow will not cause any exception,
+// nor will they saturate, rather they will 'wrap' to yield the low-order n bits of the result where
+// n is the size in bits of the integer."
+TEST_F(ConstantFoldingTest, FoldSignedIntegerDiffOverflow)
+{
+    const std::string &shaderString =
+        "#version 300 es\n"
+        "precision mediump float;\n"
+        "out vec4 my_FragColor;\n"
+        "void main()\n"
+        "{\n"
+        "    int i = -0x7fffffff - 7;\n"
+        "    my_FragColor = vec4(i);\n"
+        "}\n";
+    compile(shaderString);
+    ASSERT_TRUE(constantFoundInAST(0x7ffffffa));
+}
+
+// Test that folding an unsigned integer multiplication that overflows works.
+// ESSL 3.00.6 section 4.1.3 Integers:
+// "For all precisions, operations resulting in overflow or underflow will not cause any exception,
+// nor will they saturate, rather they will 'wrap' to yield the low-order n bits of the result where
+// n is the size in bits of the integer."
+TEST_F(ConstantFoldingTest, FoldUnsignedIntegerMultiplyOverflow)
+{
+    const std::string &shaderString =
+        "#version 300 es\n"
+        "precision mediump float;\n"
+        "out vec4 my_FragColor;\n"
+        "void main()\n"
+        "{\n"
+        "    uint u = 0xffffffffu * 10u;\n"
+        "    my_FragColor = vec4(u);\n"
+        "}\n";
+    compile(shaderString);
+    ASSERT_TRUE(constantFoundInAST(0xfffffff6u));
+}
+
+// Test that folding a signed integer multiplication that overflows works.
+// ESSL 3.00.6 section 4.1.3 Integers:
+// "For all precisions, operations resulting in overflow or underflow will not cause any exception,
+// nor will they saturate, rather they will 'wrap' to yield the low-order n bits of the result where
+// n is the size in bits of the integer."
+TEST_F(ConstantFoldingTest, FoldSignedIntegerMultiplyOverflow)
+{
+    const std::string &shaderString =
+        "#version 300 es\n"
+        "precision mediump float;\n"
+        "out vec4 my_FragColor;\n"
+        "void main()\n"
+        "{\n"
+        "    int i = 0x7fffffff * 42;\n"
+        "    my_FragColor = vec4(i);\n"
+        "}\n";
+    compile(shaderString);
+    ASSERT_TRUE(constantFoundInAST(-42));
+}

src/tests/gl_tests/GLSLTest.cpp

@@ -2368,71 +2368,6 @@ TEST_P(GLSLTest, NestedSequenceOperatorWithTernaryInside)
     EXPECT_PIXEL_COLOR_EQ(0, 0, GLColor::green);
 }
 
-// Test that multiplication ops are properly validated.
-TEST_P(GLSLTest, FoldedIntProductOutOfBounds)
-{
-    const std::string &fragmentShader =
-        "precision mediump float;\n"
-        "void main(void)\n"
-        "{\n"
-        " int prod = -2580 * 25800 * 25800;\n"
-        " gl_FragColor = vec4(float(prod));\n"
-        "}\n";
-
-    GLuint program = CompileProgram(mSimpleVSSource, fragmentShader);
-    EXPECT_EQ(0u, program);
-    glDeleteProgram(program);
-}
-
-// Test that multiplication ops are properly validated.
-TEST_P(GLSLTest_ES3, FoldedUIntProductOutOfBounds)
-{
-    const std::string &fragmentShader =
-        "#version 300 es\n"
-        "precision mediump float;\n"
-        "void main()\n"
-        "{\n"
-        "   unsigned int prod = 2580u * 25800u * 25800u;\n"
-        "   gl_FragColor = vec4(float(prod));\n"
-        "}\n";
-
-    GLuint program = CompileProgram(mSimpleVSSource, fragmentShader);
-    EXPECT_EQ(0u, program);
-    glDeleteProgram(program);
-}
-
-// Test that addition ops are properly validated.
-TEST_P(GLSLTest, FoldedIntSumOutOfBounds)
-{
-    const std::string &fragmentShader =
-        "precision mediump float;\n"
-        "void main(void)\n"
-        "{\n"
-        " int sum = 2147483647 + 2147483647;\n"
-        " gl_FragColor = vec4(float(sum));\n"
-        "}\n";
-
-    GLuint program = CompileProgram(mSimpleVSSource, fragmentShader);
-    EXPECT_EQ(0u, program);
-    glDeleteProgram(program);
-}
-
-// Test that subtraction ops are properly validated.
-TEST_P(GLSLTest, FoldedIntDifferenceOutOfBounds)
-{
-    const std::string &fragmentShader =
-        "precision mediump float;\n"
-        "void main(void)\n"
-        "{\n"
-        " int diff = -2147483000 - 2147483000;\n"
-        " gl_FragColor = vec4(float(diff));\n"
-        "}\n";
-
-    GLuint program = CompileProgram(mSimpleVSSource, fragmentShader);
-    EXPECT_EQ(0u, program);
-    glDeleteProgram(program);
-}
-
 // Test that using a sampler2D and samplerExternalOES in the same shader works (anglebug.com/1534)
 TEST_P(GLSLTest, ExternalAnd2DSampler)
 {