Only require that the stencil masks are same as many effective bits set

AUTHORS

@@ -41,3 +41,4 @@ Aitor Moreno
 Yuri O'Donnell
 Josh Soref
 Maks Naumov
+Jinyoung Hur

src/libANGLE/renderer/d3d/d3d11/Renderer11.cpp

@@ -239,7 +239,8 @@ Renderer11::Renderer11(egl::Display *display)
     : RendererD3D(display),
       mStateCache(this),
       mLastHistogramUpdateTime(ANGLEPlatformCurrent()->monotonicallyIncreasingTime()),
-      mDebug(nullptr)
+      mDebug(nullptr),
+      mCurStencilSize(0)
 {
     mVertexDataManager = NULL;
     mIndexDataManager = NULL;
@@ -1198,9 +1199,17 @@ gl::Error Renderer11::setDepthStencilState(const gl::DepthStencilState &depthSte
         memcmp(&depthStencilState, &mCurDepthStencilState, sizeof(gl::DepthStencilState)) != 0 ||
         stencilRef != mCurStencilRef || stencilBackRef != mCurStencilBackRef)
     {
-        ASSERT(depthStencilState.stencilWritemask == depthStencilState.stencilBackWritemask);
+        // get the maximum size of the stencil ref
+        unsigned int maxStencil = 0;
+        if (depthStencilState.stencilTest && mCurStencilSize > 0)
+        {
+            maxStencil = (1 << mCurStencilSize) - 1;
+        }
+        ASSERT((depthStencilState.stencilWritemask & maxStencil) ==
+               (depthStencilState.stencilBackWritemask & maxStencil));
         ASSERT(stencilRef == stencilBackRef);
-        ASSERT(depthStencilState.stencilMask == depthStencilState.stencilBackMask);
+        ASSERT((depthStencilState.stencilMask & maxStencil) ==
+               (depthStencilState.stencilBackMask & maxStencil));
 
         ID3D11DepthStencilState *dxDepthStencilState = NULL;
         gl::Error error = mStateCache.getDepthStencilState(depthStencilState, &dxDepthStencilState);
@@ -1500,6 +1509,13 @@ gl::Error Renderer11::applyRenderTarget(const gl::Framebuffer *framebuffer)
             unsetConflictingSRVs(gl::SAMPLER_VERTEX, depthStencilResource, index);
             unsetConflictingSRVs(gl::SAMPLER_PIXEL, depthStencilResource, index);
         }
+
+        unsigned int stencilSize = depthStencil->getStencilSize();
+        if (!mDepthStencilInitialized || stencilSize != mCurStencilSize)
+        {
+            mCurStencilSize            = stencilSize;
+            mForceSetDepthStencilState = true;
+        }
     }
 
     // Apply the render target and depth stencil

src/libANGLE/renderer/d3d/d3d11/Renderer11.h

@@ -323,6 +323,7 @@ class Renderer11 : public RendererD3D
     uintptr_t mAppliedDSV;
     bool mDepthStencilInitialized;
     bool mRenderTargetDescInitialized;
+    unsigned int mCurStencilSize;
 
     struct RenderTargetDesc
     {

src/libANGLE/renderer/d3d/d3d9/Renderer9.cpp

@@ -1065,13 +1065,15 @@ gl::Error Renderer9::setDepthStencilState(const gl::DepthStencilState &depthSten
             const D3DRENDERSTATETYPE D3DRS_CCW_STENCILMASK = D3DRS_STENCILMASK;
             const D3DRENDERSTATETYPE D3DRS_CCW_STENCILWRITEMASK = D3DRS_STENCILWRITEMASK;
 
-            ASSERT(depthStencilState.stencilWritemask == depthStencilState.stencilBackWritemask);
-            ASSERT(stencilRef == stencilBackRef);
-            ASSERT(depthStencilState.stencilMask == depthStencilState.stencilBackMask);
-
             // get the maximum size of the stencil ref
             unsigned int maxStencil = (1 << mCurStencilSize) - 1;
 
+            ASSERT((depthStencilState.stencilWritemask & maxStencil) ==
+                   (depthStencilState.stencilBackWritemask & maxStencil));
+            ASSERT(stencilRef == stencilBackRef);
+            ASSERT((depthStencilState.stencilMask & maxStencil) ==
+                   (depthStencilState.stencilBackMask & maxStencil));
+
             mDevice->SetRenderState(frontFaceCCW ? D3DRS_STENCILWRITEMASK : D3DRS_CCW_STENCILWRITEMASK,
                                     depthStencilState.stencilWritemask);
             mDevice->SetRenderState(frontFaceCCW ? D3DRS_STENCILFUNC : D3DRS_CCW_STENCILFUNC,

src/libANGLE/validationES.cpp

@@ -1500,10 +1500,16 @@ static bool ValidateDrawBase(Context *context, GLenum mode, GLsizei count, GLsiz
 
     if (context->getLimitations().noSeparateStencilRefsAndMasks)
     {
-        const gl::DepthStencilState &depthStencilState = state.getDepthStencilState();
-        if (depthStencilState.stencilWritemask != depthStencilState.stencilBackWritemask ||
+        const Framebuffer *framebuffer             = context->getState().getDrawFramebuffer();
+        const FramebufferAttachment *stencilBuffer = framebuffer->getStencilbuffer();
+        GLuint stencilBits                         = stencilBuffer ? stencilBuffer->getStencilSize() : 0;
+        GLuint minimumRequiredStencilMask          = (1 << stencilBits) - 1;
+        const DepthStencilState &depthStencilState = state.getDepthStencilState();
+        if ((depthStencilState.stencilWritemask & minimumRequiredStencilMask) !=
+                (depthStencilState.stencilBackWritemask & minimumRequiredStencilMask) ||
             state.getStencilRef() != state.getStencilBackRef() ||
-            depthStencilState.stencilMask != depthStencilState.stencilBackMask)
+            (depthStencilState.stencilMask & minimumRequiredStencilMask) !=
+                (depthStencilState.stencilBackMask & minimumRequiredStencilMask))
         {
             // Note: these separate values are not supported in WebGL, due to D3D's limitations. See
             // Section 6.10 of the WebGL 1.0 spec

src/tests/gl_tests/DifferentStencilMasksTest.cpp

+//
+// Copyright 2015 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// DifferentStencilMasksTest:
+//   Tests the equality between stencilWriteMask and stencilBackWriteMask.
+//
+
+#include "test_utils/ANGLETest.h"
+
+using namespace angle;
+
+namespace
+{
+class DifferentStencilMasksTest : public ANGLETest
+{
+  protected:
+    DifferentStencilMasksTest() : mProgram(0)
+    {
+        setWindowWidth(128);
+        setWindowHeight(128);
+        setConfigRedBits(8);
+        setConfigGreenBits(8);
+        setConfigBlueBits(8);
+        setConfigAlphaBits(8);
+        setConfigDepthBits(24);
+        setConfigStencilBits(8);
+    }
+
+    void SetUp() override
+    {
+        ANGLETest::SetUp();
+
+        const std::string vertexShaderSource = SHADER_SOURCE
+        (
+            precision highp float;
+            attribute vec4 position;
+
+            void main()
+            {
+                gl_Position = position;
+            }
+        );
+
+        const std::string fragmentShaderSource = SHADER_SOURCE
+        (
+            precision highp float;
+
+            void main()
+            {
+                gl_FragColor = vec4(0.0, 0.0, 1.0, 1.0);
+            }
+        );
+
+        mProgram = CompileProgram(vertexShaderSource, fragmentShaderSource);
+        ASSERT_NE(0u, mProgram);
+
+        glEnable(GL_STENCIL_TEST);
+        ASSERT_GL_NO_ERROR();
+    }
+
+    void TearDown() override
+    {
+        glDisable(GL_STENCIL_TEST);
+        if (mProgram != 0)
+            glDeleteProgram(mProgram);
+
+        ANGLETest::TearDown();
+    }
+
+    GLuint mProgram;
+};
+
+// Tests that effectively same front and back masks are legal.
+TEST_P(DifferentStencilMasksTest, DrawWithSameEffectiveMask)
+{
+    // 0x00ff and 0x01ff are effectively 0x00ff by being masked by the current stencil bits, 8.
+    glStencilMaskSeparate(GL_FRONT, 0x00ff);
+    glStencilMaskSeparate(GL_BACK, 0x01ff);
+
+    glUseProgram(mProgram);
+
+    drawQuad(mProgram, "position", 0.5f);
+
+    EXPECT_GL_NO_ERROR();
+}
+
+// Tests that effectively different front and back masks are illegal.
+TEST_P(DifferentStencilMasksTest, DrawWithDifferentMask)
+{
+    glStencilMaskSeparate(GL_FRONT, 0x0001);
+    glStencilMaskSeparate(GL_BACK, 0x0002);
+
+    glUseProgram(mProgram);
+
+    drawQuad(mProgram, "position", 0.5f);
+
+    EXPECT_GL_ERROR(GL_INVALID_OPERATION);
+}
+
+class DifferentStencilMasksWithoutStencilBufferTest : public ANGLETest
+{
+  protected:
+    DifferentStencilMasksWithoutStencilBufferTest() : mProgram(0)
+    {
+        setWindowWidth(128);
+        setWindowHeight(128);
+        setConfigRedBits(8);
+        setConfigGreenBits(8);
+        setConfigBlueBits(8);
+        setConfigAlphaBits(8);
+        setConfigDepthBits(0);
+        setConfigStencilBits(0);
+    }
+
+    void SetUp() override
+    {
+        ANGLETest::SetUp();
+
+        const std::string vertexShaderSource = SHADER_SOURCE
+        (
+            precision highp float;
+            attribute vec4 position;
+
+            void main()
+            {
+                gl_Position = position;
+            }
+        );
+
+        const std::string fragmentShaderSource = SHADER_SOURCE
+        (
+            precision highp float;
+
+            void main()
+            {
+                gl_FragColor = vec4(0.0, 0.0, 1.0, 1.0);
+            }
+        );
+
+        mProgram = CompileProgram(vertexShaderSource, fragmentShaderSource);
+        ASSERT_NE(0u, mProgram);
+
+        glEnable(GL_STENCIL_TEST);
+        ASSERT_GL_NO_ERROR();
+    }
+
+    void TearDown() override
+    {
+        glDisable(GL_STENCIL_TEST);
+        if (mProgram != 0)
+            glDeleteProgram(mProgram);
+
+        ANGLETest::TearDown();
+    }
+
+    GLuint mProgram;
+};
+
+// Tests that effectively different front and back masks, without stencil bits, are legal.
+TEST_P(DifferentStencilMasksWithoutStencilBufferTest, DrawWithDifferentMask)
+{
+    glStencilMaskSeparate(GL_FRONT, 0x0001);
+    glStencilMaskSeparate(GL_BACK, 0x0002);
+
+    glUseProgram(mProgram);
+
+    drawQuad(mProgram, "position", 0.5f);
+
+    EXPECT_GL_NO_ERROR();
+}
+
+// Use this to select which configurations (e.g. which renderer, which GLES major version) these
+// tests should be run against.
+ANGLE_INSTANTIATE_TEST(DifferentStencilMasksTest, ES2_D3D9(), ES2_D3D11(), ES3_D3D11(), ES2_OPENGL(), ES3_OPENGL());
+ANGLE_INSTANTIATE_TEST(DifferentStencilMasksWithoutStencilBufferTest, ES2_D3D9(), ES2_D3D11(), ES3_D3D11(), ES2_OPENGL(), ES3_OPENGL());
+} // anonymous namespace