Precision fix for inversesqrt.

Fixes all dEQP failures in inversesqrt and normalize. Change-Id: Ifd1ac2714fa95bab1c720b65c95a27bbf23ea933 Reviewed-on: https://swiftshader-review.googlesource.com/13728Tested-by: Alexis Hétu <sugoi@google.com> Reviewed-by: Nicolas Capens <nicolascapens@google.com>

Precision fix for inversesqrt.
a0ef97a3 · Alexis Hetu · Alexis Hétu · 0b7003bd · a0ef97a3
Commit a0ef97a3 authored Nov 13, 2017 by Alexis Hetu Committed by Alexis Hétu Nov 14, 2017
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 4 deletions

ShaderCore.cpp src/Shader/ShaderCore.cpp +3 -4

No files found.
--- a/src/Shader/ShaderCore.cpp
+++ b/src/Shader/ShaderCore.cpp
@@ -227,7 +227,7 @@ namespace sw
 		Float4 rsq;
-		if(!pp && rsqPrecision >= IEEE)
+		if(!pp)
 		{
 			rsq = Float4(1.0f) / Sqrt(abs);
 		}
@@ -239,10 +239,9 @@ namespace sw
 			{
 				rsq = rsq * (Float4(3.0f) - rsq * rsq * abs) * Float4(0.5f);
 			}
-		}
-		int big = 0x7F7FFFFF;
+			rsq = As<Float4>(CmpNEQ(As<Int4>(abs), Int4(0x7F800000)) & As<Int4>(rsq));
-		rsq = Min(rsq, Float4((float&)big));
+		}
 		return rsq;
 	}