Fix Frac() returning 1.0.

Frac() should always produce results in the range [0.0, 1.0), and thus never produce a 1.0. However, the current implementation uses x - floor(x) and this returns 1.0 for very small negative values due to catastrophic cancellation. Bug swiftshader:74 Change-Id: I942dd7cfb1f7ee3a260070e748704f005eed0b13 Reviewed-on: https://swiftshader-review.googlesource.com/10648Tested-by: Nicolas Capens <nicolascapens@google.com> Reviewed-by: Alexis Hétu <sugoi@google.com>

Fix Frac() returning 1.0.
b923042e · Nicolas Capens · Nicolas Capens · ab849aec · b923042e · b923042e
Commit b923042e authored Jul 17, 2017 by Nicolas Capens Committed by Nicolas Capens Jul 17, 2017
Hide whitespace changes
Inline Side-by-side

Showing with 18 additions and 6 deletions

LLVMReactor.cpp src/Reactor/LLVMReactor.cpp +9 -3

SubzeroReactor.cpp src/Reactor/SubzeroReactor.cpp +9 -3

No files found.
--- a/src/Reactor/LLVMReactor.cpp
+++ b/src/Reactor/LLVMReactor.cpp
@@ -6255,16 +6255,22 @@ namespace sw

 	RValue<Float4> Frac(RValue<Float4> x)
 	{
+		Float4 frc;
+
 		if(CPUID::supportsSSE4_1())
 		{
-			return x - x86::floorps(x);
+			frc = x - x86::floorps(x);
 		}
 		else
 		{
-			Float4 frc = x - Float4(Int4(x));   // Signed fractional part
+			frc = x - Float4(Int4(x));   // Signed fractional part.

-			return frc + As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1, 1, 1, 1)));
+			frc += As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1.0f)));   // Add 1.0 if negative.
 		}
+
+		// x - floor(x) can be 1.0 for very small negative x.
+		// Clamp against the value just below 1.0.
+		return Min(frc, As<Float4>(Int4(0x3F7FFFFF)));
 	}

 	RValue<Float4> Floor(RValue<Float4> x)

--- a/src/Reactor/SubzeroReactor.cpp
+++ b/src/Reactor/SubzeroReactor.cpp
@@ -6325,16 +6325,22 @@ namespace sw

 	RValue<Float4> Frac(RValue<Float4> x)
 	{
+		Float4 frc;
+
 		if(CPUID::SSE4_1)
 		{
-			return x - Floor(x);
+			frc = x - Floor(x);
 		}
 		else
 		{
-			Float4 frc = x - Float4(Int4(x));   // Signed fractional part
+			frc = x - Float4(Int4(x));   // Signed fractional part.

-			return frc + As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1, 1, 1, 1)));
+			frc += As<Float4>(As<Int4>(CmpNLE(Float4(0.0f), frc)) & As<Int4>(Float4(1, 1, 1, 1)));   // Add 1.0 if negative.
 		}
+
+		// x - floor(x) can be 1.0 for very small negative x.
+		// Clamp against the value just below 1.0.
+		return Min(frc, As<Float4>(Int4(0x3F7FFFFF)));
 	}

 	RValue<Float4> Floor(RValue<Float4> x)