Skip to content

S
swiftshader
Commit 0fc611f1 by Ben Clayton
Options

Reactor: Add Gather and Scatter instructions.

Use these for a fast-path for Load() and Store(). This is an attempt to fix the severe performance hit we incurred with robustness. Bug: b/131224163 Change-Id: I3e244bed5ed723cf29538ff022781c813caaa5eb Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/29454Tested-by: 's avatarBen Clayton <bclayton@google.com> Presubmit-Ready: Ben Clayton <bclayton@google.com> Kokoro-Presubmit: kokoro <noreply+kokoro@google.com> Reviewed-by: 's avatarNicolas Capens <nicolascapens@google.com>
parent 9e4bc1ba
Showing with 140 additions and 33 deletions
src/Pipeline/SpirvShader.cpp
......@@ -239,37 +239,44 @@ namespace sw
T Load(Pointer ptr, Int mask, bool atomic /* = false */, std::memory_order order /* = std::memory_order_relaxed */)
{
using EL = typename Element<T>::type;
T out;
auto offsets = ptr.offsets();
mask &= CmpLT(offsets + SIMD::Int(sizeof(float) - 1), SIMD::Int(ptr.limit)); // Disable OOB reads.
auto anyLanesDisabled = AnyFalse(mask);
If(ptr.hasEqualOffsets() && !anyLanesDisabled)
if (!atomic && order == std::memory_order_relaxed)
{
// Load one, replicate.
auto offset = Extract(offsets, 0);
out = T(Load(rr::Pointer<EL>(&ptr.base[offset]), sizeof(float), atomic, order));
return rr::Gather(rr::Pointer<EL>(ptr.base), offsets, mask, sizeof(float));
}
Else If(ptr.hasSequentialOffsets() && !anyLanesDisabled)
{
// Load all elements in a single SIMD instruction.
auto offset = Extract(offsets, 0);
out = Load(rr::Pointer<T>(&ptr.base[offset]), sizeof(float), atomic, order);
}
Else
else
{
// Divergent offsets or masked lanes - load each element individually.
out = T(0);
for (int i = 0; i < SIMD::Width; i++)
T out;
auto anyLanesDisabled = AnyFalse(mask);
If(ptr.hasEqualOffsets() && !anyLanesDisabled)
{
// Load one, replicate.
auto offset = Extract(offsets, 0);
out = T(rr::Load(rr::Pointer<EL>(&ptr.base[offset]), sizeof(float), atomic, order));
}
Else If(ptr.hasSequentialOffsets() && !anyLanesDisabled)
{
// Load all elements in a single SIMD instruction.
auto offset = Extract(offsets, 0);
out = rr::Load(rr::Pointer<T>(&ptr.base[offset]), sizeof(float), atomic, order);
}
Else
{
If(Extract(mask, i) != 0)
// Divergent offsets or masked lanes.
out = T(0);
for (int i = 0; i < SIMD::Width; i++)
{
auto offset = Extract(offsets, i);
auto el = rr::Load(rr::Pointer<EL>(&ptr.base[offset]), sizeof(float), atomic, order);
out = Insert(out, el, i);
If(Extract(mask, i) != 0)
{
auto offset = Extract(offsets, i);
auto el = rr::Load(rr::Pointer<EL>(&ptr.base[offset]), sizeof(float), atomic, order);
out = Insert(out, el, i);
}
}
}
return out;
}
return out;
}
template<typename T>
......@@ -278,22 +285,29 @@ namespace sw
using EL = typename Element<T>::type;
auto offsets = ptr.offsets();
mask &= CmpLT(offsets + SIMD::Int(sizeof(float) - 1), SIMD::Int(ptr.limit)); // Disable OOB reads.
auto anyLanesDisabled = AnyFalse(mask);
If(ptr.hasSequentialOffsets() && !anyLanesDisabled)
if (!atomic && order == std::memory_order_relaxed)
{
// Store all elements in a single SIMD instruction.
auto offset = Extract(offsets, 0);
Store(val, rr::Pointer<T>(&ptr.base[offset]), sizeof(float), atomic, order);
return rr::Scatter(rr::Pointer<EL>(ptr.base), val, offsets, mask, sizeof(float));
}
Else
else
{
// Divergent offsets or masked lanes.
for (int i = 0; i < SIMD::Width; i++)
auto anyLanesDisabled = AnyFalse(mask);
If(ptr.hasSequentialOffsets() && !anyLanesDisabled)
{
If(Extract(mask, i) != 0)
// Store all elements in a single SIMD instruction.
auto offset = Extract(offsets, 0);
Store(val, rr::Pointer<T>(&ptr.base[offset]), sizeof(float), atomic, order);
}
Else
{
// Divergent offsets or masked lanes.
for (int i = 0; i < SIMD::Width; i++)
{
auto offset = Extract(offsets, i);
rr::Store(Extract(val, i), rr::Pointer<EL>(&ptr.base[offset]), sizeof(float), atomic, order);
If(Extract(mask, i) != 0)
{
auto offset = Extract(offsets, i);
rr::Store(Extract(val, i), rr::Pointer<EL>(&ptr.base[offset]), sizeof(float), atomic, order);
}
}
}
}
......
src/Reactor/LLVMReactor.cpp
......@@ -882,7 +882,17 @@ namespace rr
#error "unknown architecture"
#endif
llvm::SmallVector<std::string, 1> mattrs;
llvm::SmallVector<std::string, 8> mattrs;
llvm::StringMap<bool> features;
bool ok = llvm::sys::getHostCPUFeatures(features);
ASSERT_MSG(ok, "llvm::sys::getHostCPUFeatures returned false");
for (auto &feature : features)
{
if (feature.second) { mattrs.push_back(feature.first()); }
}
#if 0
#if defined(__i386__) || defined(__x86_64__)
mattrs.push_back(CPUID::supportsMMX() ? "+mmx" : "-mmx");
mattrs.push_back(CPUID::supportsCMOV() ? "+cmov" : "-cmov");
......@@ -899,6 +909,7 @@ namespace rr
// might fail to link.
#endif
#endif
#endif
llvm::TargetOptions targetOpts;
targetOpts.UnsafeFPMath = false;
......@@ -1299,6 +1310,55 @@ namespace rr
}
}
Value *Nucleus::createGather(Value *base, Type *elTy, Value *offsets, Value *mask, unsigned int alignment)
{
ASSERT(V(base)->getType()->isPointerTy());
ASSERT(V(offsets)->getType()->isVectorTy());
ASSERT(V(mask)->getType()->isVectorTy());
auto numEls = V(mask)->getType()->getVectorNumElements();
auto i1Ty = ::llvm::Type::getInt1Ty(*::context);
auto i32Ty = ::llvm::Type::getInt32Ty(*::context);
auto i8Ty = ::llvm::Type::getInt8Ty(*::context);
auto i8PtrTy = i8Ty->getPointerTo();
auto elPtrTy = T(elTy)->getPointerTo();
auto elVecTy = ::llvm::VectorType::get(T(elTy), numEls);
auto elPtrVecTy = ::llvm::VectorType::get(elPtrTy, numEls);
auto i8Base = ::builder->CreatePointerCast(V(base), i8PtrTy);
auto i8Ptrs = ::builder->CreateGEP(i8Base, V(offsets));
auto elPtrs = ::builder->CreatePointerCast(i8Ptrs, elPtrVecTy);
auto i8Mask = ::builder->CreateIntCast(V(mask), ::llvm::VectorType::get(i1Ty, numEls), false); // vec<int, int, ...> -> vec<bool, bool, ...>
auto passthrough = ::llvm::Constant::getNullValue(elVecTy);
auto align = ::llvm::ConstantInt::get(i32Ty, alignment);
auto func = ::llvm::Intrinsic::getDeclaration(::module, llvm::Intrinsic::masked_gather, { elVecTy, elPtrVecTy } );
return V(::builder->CreateCall(func, { elPtrs, align, i8Mask, passthrough }));
}
void Nucleus::createScatter(Value *base, Value *val, Value *offsets, Value *mask, unsigned int alignment)
{
ASSERT(V(base)->getType()->isPointerTy());
ASSERT(V(val)->getType()->isVectorTy());
ASSERT(V(offsets)->getType()->isVectorTy());
ASSERT(V(mask)->getType()->isVectorTy());
auto numEls = V(mask)->getType()->getVectorNumElements();
auto i1Ty = ::llvm::Type::getInt1Ty(*::context);
auto i32Ty = ::llvm::Type::getInt32Ty(*::context);
auto i8Ty = ::llvm::Type::getInt8Ty(*::context);
auto i8PtrTy = i8Ty->getPointerTo();
auto elVecTy = V(val)->getType();
auto elTy = elVecTy->getVectorElementType();
auto elPtrTy = elTy->getPointerTo();
auto elPtrVecTy = ::llvm::VectorType::get(elPtrTy, numEls);
auto i8Base = ::builder->CreatePointerCast(V(base), i8PtrTy);
auto i8Ptrs = ::builder->CreateGEP(i8Base, V(offsets));
auto elPtrs = ::builder->CreatePointerCast(i8Ptrs, elPtrVecTy);
auto i8Mask = ::builder->CreateIntCast(V(mask), ::llvm::VectorType::get(i1Ty, numEls), false); // vec<int, int, ...> -> vec<bool, bool, ...>
auto align = ::llvm::ConstantInt::get(i32Ty, alignment);
auto func = ::llvm::Intrinsic::getDeclaration(::module, llvm::Intrinsic::masked_scatter, { elVecTy, elPtrVecTy } );
::builder->CreateCall(func, { V(val), elPtrs, align, i8Mask });
}
Value *Nucleus::createGEP(Value *ptr, Type *type, Value *index, bool unsignedIndex)
{
RR_DEBUG_INFO_UPDATE_LOC();
......
src/Reactor/Nucleus.hpp
......@@ -100,6 +100,10 @@ namespace rr
static Value *createStore(Value *value, Value *ptr, Type *type, bool isVolatile = false, unsigned int aligment = 0, bool atomic = false, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createGEP(Value *ptr, Type *type, Value *index, bool unsignedIndex);
// Scatter / Gather instructions
static Value *createGather(Value *base, Type *elementType, Value *offsets, Value *mask, unsigned int alignment);
static void createScatter(Value *base, Value *value, Value *offsets, Value *mask, unsigned int alignment);
// Atomic instructions
static Value *createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createAtomicSub(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
......
src/Reactor/Reactor.cpp
......@@ -4212,4 +4212,25 @@ namespace rr
Nucleus::createCondBr(cmp.value, bodyBB, endBB);
Nucleus::setInsertBlock(bodyBB);
}
RValue<Float4> Gather(RValue<Pointer<Float>> base, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment)
{
return RValue<Float4>(Nucleus::createGather(base.value, Float::getType(), offsets.value, mask.value, alignment));
}
RValue<Int4> Gather(RValue<Pointer<Int>> base, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment)
{
return RValue<Int4>(Nucleus::createGather(base.value, Int::getType(), offsets.value, mask.value, alignment));
}
void Scatter(RValue<Pointer<Float>> base, RValue<Float4> val, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment)
{
Nucleus::createScatter(base.value, val.value, offsets.value, mask.value, alignment);
}
void Scatter(RValue<Pointer<Int>> base, RValue<Int4> val, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment)
{
Nucleus::createScatter(base.value, val.value, offsets.value, mask.value, alignment);
}
}
src/Reactor/Reactor.hpp
......@@ -2355,6 +2355,12 @@ namespace rr
return Load(RValue<Pointer<T>>(pointer), alignment, atomic, memoryOrder);
}
// TODO: Use SIMD to template these.
RValue<Float4> Gather(RValue<Pointer<Float>> base, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment);
RValue<Int4> Gather(RValue<Pointer<Int>> base, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment);
void Scatter(RValue<Pointer<Float>> base, RValue<Float4> val, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment);
void Scatter(RValue<Pointer<Int>> base, RValue<Int4> val, RValue<Int4> offsets, RValue<Int4> mask, unsigned int alignment);
template<typename T>
void Store(RValue<T> value, RValue<Pointer<T>> pointer, unsigned int alignment, bool atomic, std::memory_order memoryOrder)
{
......
src/Reactor/SubzeroReactor.cpp
......@@ -3453,6 +3453,8 @@ namespace rr
// Below are functions currently unimplemented for the Subzero backend.
// They are stubbed to satisfy the linker.
Value *Nucleus::createGather(Value *base, Type *elTy, Value *offsets, Value *mask, unsigned int alignment) { UNIMPLEMENTED("Subzero createGather()"); return nullptr; }
void Nucleus::createScatter(Value *base, Value *val, Value *offsets, Value *mask, unsigned int alignment) { UNIMPLEMENTED("Subzero createScatter()"); }
RValue<Float4> Sin(RValue<Float4> x) { UNIMPLEMENTED("Subzero Sin()"); return Float4(0); }
RValue<Float4> Cos(RValue<Float4> x) { UNIMPLEMENTED("Subzero Cos()"); return Float4(0); }
RValue<Float4> Tan(RValue<Float4> x) { UNIMPLEMENTED("Subzero Tan()"); return Float4(0); }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment