According to spec, the stencil ref is clamped to the range [0,2^s-1]

The clamp to zero occurs on set since negative values will never be useful. The clamp to 2^s - 1 occurs on use since it is dependant on the currently bound stencil buffer Original-Author: Jim Hauxwell <james@dattrax.co.uk> Signed-off-by: Daniel Koch git-svn-id: https://angleproject.googlecode.com/svn/trunk@324 736b8ea6-26fd-11df-bfd4-992fa37f6226

According to spec, the stencil ref is clamped to the range [0,2^s-1]
dd7948b3 · daniel@transgaming.com · 91b72320 · dd7948b3
Commit dd7948b3 authored Jun 02, 2010 by daniel@transgaming.com
Hide whitespace changes
Inline Side-by-side

Showing with 9 additions and 4 deletions

Context.cpp src/libGLESv2/Context.cpp +9 -4

No files found.
--- a/src/libGLESv2/Context.cpp
+++ b/src/libGLESv2/Context.cpp
@@ -454,7 +454,7 @@ void Context::setStencilParams(GLenum stencilFunc, GLint stencilRef, GLuint sten
        mState.stencilMask != stencilMask)
    {
        mState.stencilFunc = stencilFunc;
-        mState.stencilRef = stencilRef;
+        mState.stencilRef = (stencilRef > 0) ? stencilRef : 0;
        mState.stencilMask = stencilMask;
        mStencilStateDirty = true;
    }
@@ -467,7 +467,7 @@ void Context::setStencilBackParams(GLenum stencilBackFunc, GLint stencilBackRef,
        mState.stencilBackMask != stencilBackMask)
    {
        mState.stencilBackFunc = stencilBackFunc;
-        mState.stencilBackRef = stencilBackRef;
+        mState.stencilBackRef = (stencilBackRef > 0) ? stencilBackRef : 0;
        mState.stencilBackMask = stencilBackMask;
        mStencilStateDirty = true;
    }
@@ -1840,11 +1840,16 @@ void Context::applyState(GLenum drawMode)
                return error(GL_INVALID_OPERATION);
            }

+            // get the maximum size of the stencil ref
+            gl::Framebuffer *framebuffer = getFramebuffer();
+            gl::Stencilbuffer *stencilbuffer = framebuffer->getStencilbuffer();
+            GLuint maxStencil = (1 << stencilbuffer->getStencilSize()) - 1;
+
            device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILWRITEMASK : D3DRS_CCW_STENCILWRITEMASK, mState.stencilWritemask);
            device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILFUNC : D3DRS_CCW_STENCILFUNC, 
                                   es2dx::ConvertComparison(mState.stencilFunc));

-            device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILREF : D3DRS_CCW_STENCILREF, mState.stencilRef);   // FIXME: Clamp to range
+            device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILREF : D3DRS_CCW_STENCILREF, (mState.stencilRef < (GLint)maxStencil) ? mState.stencilRef : maxStencil);
            device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILMASK : D3DRS_CCW_STENCILMASK, mState.stencilMask);

            device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILFAIL : D3DRS_CCW_STENCILFAIL, 
@@ -1858,7 +1863,7 @@ void Context::applyState(GLenum drawMode)
            device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILFUNC : D3DRS_CCW_STENCILFUNC, 
                                   es2dx::ConvertComparison(mState.stencilBackFunc));

-            device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILREF : D3DRS_CCW_STENCILREF, mState.stencilBackRef);   // FIXME: Clamp to range
+            device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILREF : D3DRS_CCW_STENCILREF, (mState.stencilBackRef < (GLint)maxStencil) ? mState.stencilBackRef : maxStencil);
            device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILMASK : D3DRS_CCW_STENCILMASK, mState.stencilBackMask);

            device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILFAIL : D3DRS_CCW_STENCILFAIL,