Update stencil validation rules for WebGL

src/libANGLE/Framebuffer.cpp

@@ -1838,18 +1838,24 @@ bool Framebuffer::formsRenderingFeedbackLoopWith(const State &state) const
         }
     }
 
-    // Note: we assume the front and back masks are the same for WebGL.
     const FramebufferAttachment *stencil = getStencilbuffer();
-    ASSERT(dsState.stencilBackWritemask == dsState.stencilWritemask);
-    if (stencil && stencil->type() == GL_TEXTURE && dsState.stencilTest &&
-        dsState.stencilWritemask != 0)
-    {
-        // Skip the feedback loop check if depth/stencil point to the same resource.
-        if (!depth || *stencil != *depth)
+    if (dsState.stencilTest && stencil)
+    {
+        GLuint stencilSize = stencil->getStencilSize();
+        ASSERT(stencilSize <= 8);
+        GLuint maxStencilValue = (1 << stencilSize) - 1;
+        // We assume the front and back masks are the same for WebGL.
+        ASSERT((dsState.stencilBackWritemask & maxStencilValue) ==
+               (dsState.stencilWritemask & maxStencilValue));
+        if (stencil->type() == GL_TEXTURE && dsState.stencilWritemask != 0)
         {
-            if (program->samplesFromTexture(state, stencil->id()))
+            // Skip the feedback loop check if depth/stencil point to the same resource.
+            if (!depth || *stencil != *depth)
             {
-                return true;
+                if (program->samplesFromTexture(state, stencil->id()))
+                {
+                    return true;
+                }
             }
         }
     }

src/libANGLE/renderer/d3d/d3d11/StateManager11.cpp

@@ -9,6 +9,7 @@
 #include "libANGLE/renderer/d3d/d3d11/StateManager11.h"
 
 #include "common/bitset_utils.h"
+#include "common/mathutil.h"
 #include "common/utilities.h"
 #include "libANGLE/Context.h"
 #include "libANGLE/Query.h"
@@ -1154,7 +1155,8 @@ gl::Error StateManager11::syncDepthStencilState(const gl::State &glState)
     }
     ASSERT((mCurDepthStencilState.stencilWritemask & maxStencil) ==
            (mCurDepthStencilState.stencilBackWritemask & maxStencil));
-    ASSERT(mCurStencilRef == mCurStencilBackRef);
+    ASSERT(gl::clamp(mCurStencilRef, 0, static_cast<int>(maxStencil)) ==
+           gl::clamp(mCurStencilBackRef, 0, static_cast<int>(maxStencil)));
     ASSERT((mCurDepthStencilState.stencilMask & maxStencil) ==
            (mCurDepthStencilState.stencilBackMask & maxStencil));
 
@@ -1170,11 +1172,20 @@ gl::Error StateManager11::syncDepthStencilState(const gl::State &glState)
 
     if (mCurDisableStencil.value())
     {
-        modifiedGLState.stencilWritemask     = 0;
+        modifiedGLState.stencilTest = false;
+    }
+    if (!modifiedGLState.stencilTest)
+    {
+        modifiedGLState.stencilWritemask = 0;
         modifiedGLState.stencilBackWritemask = 0;
-        modifiedGLState.stencilTest          = false;
     }
 
+    // If STENCIL_TEST is disabled in glState, stencil testing and writing should be disabled.
+    // Verify that's true in the modifiedGLState so it is propagated to d3dState.
+    ASSERT(glState.getDepthStencilState().stencilTest ||
+           (!modifiedGLState.stencilTest && modifiedGLState.stencilWritemask == 0 &&
+            modifiedGLState.stencilBackWritemask == 0));
+
     const d3d11::DepthStencilState *d3dState = nullptr;
     ANGLE_TRY(mRenderer->getDepthStencilState(modifiedGLState, &d3dState));
     ASSERT(d3dState);
@@ -1187,7 +1198,7 @@ gl::Error StateManager11::syncDepthStencilState(const gl::State &glState)
                   "Unexpected value of D3D11_DEFAULT_STENCIL_READ_MASK");
     static_assert(D3D11_DEFAULT_STENCIL_WRITE_MASK == 0xFF,
                   "Unexpected value of D3D11_DEFAULT_STENCIL_WRITE_MASK");
-    UINT dxStencilRef = std::min<UINT>(mCurStencilRef, 0xFFu);
+    UINT dxStencilRef = static_cast<UINT>(gl::clamp(mCurStencilRef, 0, 0xFF));
 
     mRenderer->getDeviceContext()->OMSetDepthStencilState(d3dState->get(), dxStencilRef);
 

src/libANGLE/validationES.cpp

@@ -2515,28 +2515,35 @@ bool ValidateDrawBase(Context *context, GLenum mode, GLsizei count)
     Framebuffer *framebuffer = state.getDrawFramebuffer();
     if (context->getLimitations().noSeparateStencilRefsAndMasks || extensions.webglCompatibility)
     {
+        ASSERT(framebuffer);
         const FramebufferAttachment *dsAttachment =
             framebuffer->getStencilOrDepthStencilAttachment();
-        GLuint stencilBits                = dsAttachment ? dsAttachment->getStencilSize() : 0;
-        GLuint minimumRequiredStencilMask = (1 << stencilBits) - 1;
+        const GLuint stencilBits = dsAttachment ? dsAttachment->getStencilSize() : 0;
+        ASSERT(stencilBits <= 8);
+
         const DepthStencilState &depthStencilState = state.getDepthStencilState();
+        if (depthStencilState.stencilTest && stencilBits > 0)
+        {
+            GLuint maxStencilValue = (1 << stencilBits) - 1;
 
-        bool differentRefs = state.getStencilRef() != state.getStencilBackRef();
-        bool differentWritemasks =
-            (depthStencilState.stencilWritemask & minimumRequiredStencilMask) !=
-            (depthStencilState.stencilBackWritemask & minimumRequiredStencilMask);
-        bool differentMasks = (depthStencilState.stencilMask & minimumRequiredStencilMask) !=
-                              (depthStencilState.stencilBackMask & minimumRequiredStencilMask);
+            bool differentRefs =
+                clamp(state.getStencilRef(), 0, static_cast<GLint>(maxStencilValue)) !=
+                clamp(state.getStencilBackRef(), 0, static_cast<GLint>(maxStencilValue));
+            bool differentWritemasks = (depthStencilState.stencilWritemask & maxStencilValue) !=
+                                       (depthStencilState.stencilBackWritemask & maxStencilValue);
+            bool differentMasks = (depthStencilState.stencilMask & maxStencilValue) !=
+                                  (depthStencilState.stencilBackMask & maxStencilValue);
 
-        if (differentRefs || differentWritemasks || differentMasks)
-        {
-            if (!extensions.webglCompatibility)
+            if (differentRefs || differentWritemasks || differentMasks)
             {
-                ERR() << "This ANGLE implementation does not support separate front/back stencil "
-                         "writemasks, reference values, or stencil mask values.";
+                if (!extensions.webglCompatibility)
+                {
+                    ERR() << "This ANGLE implementation does not support separate front/back "
+                             "stencil writemasks, reference values, or stencil mask values.";
+                }
+                ANGLE_VALIDATION_ERR(context, InvalidOperation(), StencilReferenceMaskOrMismatch);
+                return false;
             }
-            ANGLE_VALIDATION_ERR(context, InvalidOperation(), StencilReferenceMaskOrMismatch);
-            return false;
         }
     }
 

src/tests/gl_tests/WebGLCompatibilityTest.cpp

@@ -197,6 +197,7 @@ class WebGLCompatibilityTest : public ANGLETest
                              glCheckFramebufferStatus(GL_FRAMEBUFFER));
             return;
         }
+
         ASSERT_GLENUM_EQ(GL_FRAMEBUFFER_COMPLETE, glCheckFramebufferStatus(GL_FRAMEBUFFER));
 
         const std::string renderingVs =
@@ -225,6 +226,8 @@ class WebGLCompatibilityTest : public ANGLETest
             0, 0, GLColor32F(floatData[0], floatData[1], floatData[2], floatData[3]), 1.0f);
     }
 
+    void TestDifferentStencilMaskAndRef(GLenum errIfMismatch);
+
     // Called from RenderingFeedbackLoopWithDrawBuffersEXT.
     void drawBuffersEXTFeedbackLoop(GLuint program,
                                     const std::array<GLenum, 2> &drawBuffers,
@@ -1182,8 +1185,9 @@ TEST_P(WebGLCompatibilityTest, NullPixelDataForSubImage)
     }
 }
 
-// Tests the WebGL requirement of having the same stencil mask, writemask and ref for fron and back
-TEST_P(WebGLCompatibilityTest, RequiresSameStencilMaskAndRef)
+// Tests the WebGL requirement of having the same stencil mask, writemask and ref for front and back
+// (when stencil testing is enabled)
+void WebGLCompatibilityTest::TestDifferentStencilMaskAndRef(GLenum errIfMismatch)
 {
     // Run the test in an FBO to make sure we have some stencil bits.
     GLRenderbuffer renderbuffer;
@@ -1209,7 +1213,7 @@ TEST_P(WebGLCompatibilityTest, RequiresSameStencilMaskAndRef)
     // Having a different front - back write mask generates an error.
     glStencilMaskSeparate(GL_FRONT, 1);
     glDrawArrays(GL_TRIANGLES, 0, 6);
-    EXPECT_GL_ERROR(GL_INVALID_OPERATION);
+    EXPECT_GL_ERROR(errIfMismatch);
 
     // Setting both write masks separately to the same value is valid.
     glStencilMaskSeparate(GL_BACK, 1);
@@ -1219,7 +1223,7 @@ TEST_P(WebGLCompatibilityTest, RequiresSameStencilMaskAndRef)
     // Having a different stencil front - back mask generates an error
     glStencilFuncSeparate(GL_FRONT, GL_ALWAYS, 0, 1);
     glDrawArrays(GL_TRIANGLES, 0, 6);
-    EXPECT_GL_ERROR(GL_INVALID_OPERATION);
+    EXPECT_GL_ERROR(errIfMismatch);
 
     // Setting both masks separately to the same value is valid.
     glStencilFuncSeparate(GL_BACK, GL_ALWAYS, 0, 1);
@@ -1229,7 +1233,7 @@ TEST_P(WebGLCompatibilityTest, RequiresSameStencilMaskAndRef)
     // Having a different stencil front - back reference generates an error
     glStencilFuncSeparate(GL_FRONT, GL_ALWAYS, 255, 1);
     glDrawArrays(GL_TRIANGLES, 0, 6);
-    EXPECT_GL_ERROR(GL_INVALID_OPERATION);
+    EXPECT_GL_ERROR(errIfMismatch);
 
     // Setting both references separately to the same value is valid.
     glStencilFuncSeparate(GL_BACK, GL_ALWAYS, 255, 1);
@@ -1241,6 +1245,16 @@ TEST_P(WebGLCompatibilityTest, RequiresSameStencilMaskAndRef)
     glDrawArrays(GL_TRIANGLES, 0, 6);
     ASSERT_GL_NO_ERROR();
 }
+TEST_P(WebGLCompatibilityTest, StencilTestEnabledDisallowsDifferentStencilMaskAndRef)
+{
+    glEnable(GL_STENCIL_TEST);
+    TestDifferentStencilMaskAndRef(GL_INVALID_OPERATION);
+}
+TEST_P(WebGLCompatibilityTest, StencilTestDisabledAllowsDifferentStencilMaskAndRef)
+{
+    glDisable(GL_STENCIL_TEST);
+    TestDifferentStencilMaskAndRef(GL_NO_ERROR);
+}
 
 // Test that GL_FIXED is forbidden
 TEST_P(WebGLCompatibilityTest, ForbidsGLFixed)