Skip to content

S
swiftshader
Commit 17813930 by Chris Forbes
Options

Add support atomic instructions as used with storage images

- OpAtomicIAdd - OpAtomicSMin - OpAtomicSMax - OpAtomicUMin - OpAtomicUMax - OpAtomicAnd - OpAtomicOr - OpAtomicXor - OpAtomicExchange Includes required plumbing down through Reactor. Subzero support is just stubbed for now. Bug: b/130768731 Bug: b/127472316 Test: dEQP-VK.image.* Change-Id: Iecf9e9ed602c4fde674f54491658048c96fd02c6 Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/29453Tested-by: 's avatarChris Forbes <chrisforbes@google.com> Presubmit-Ready: Chris Forbes <chrisforbes@google.com> Kokoro-Presubmit: kokoro <noreply+kokoro@google.com> Reviewed-by: 's avatarBen Clayton <bclayton@google.com> Reviewed-by: 's avatarNicolas Capens <nicolascapens@google.com>
parent b51f2c17
Showing with 222 additions and 5 deletions
src/Pipeline/SpirvShader.cpp
......@@ -678,6 +678,15 @@ namespace sw
case spv::OpDPdyFine:
case spv::OpFwidthFine:
case spv::OpAtomicLoad:
case spv::OpAtomicIAdd:
case spv::OpAtomicSMin:
case spv::OpAtomicSMax:
case spv::OpAtomicUMin:
case spv::OpAtomicUMax:
case spv::OpAtomicAnd:
case spv::OpAtomicOr:
case spv::OpAtomicXor:
case spv::OpAtomicExchange:
case spv::OpPhi:
case spv::OpImageSampleImplicitLod:
case spv::OpImageQuerySize:
......@@ -1978,6 +1987,17 @@ namespace sw
case spv::OpAtomicStore:
return EmitStore(insn, state);
case spv::OpAtomicIAdd:
case spv::OpAtomicSMin:
case spv::OpAtomicSMax:
case spv::OpAtomicUMin:
case spv::OpAtomicUMax:
case spv::OpAtomicAnd:
case spv::OpAtomicOr:
case spv::OpAtomicXor:
case spv::OpAtomicExchange:
return EmitAtomicOp(insn, state);
case spv::OpAccessChain:
case spv::OpInBoundsAccessChain:
return EmitAccessChain(insn, state);
......@@ -4695,13 +4715,75 @@ namespace sw
state->routine->createPointer(resultId, imageBase);
SIMD::Int texelOffset = GetTexelOffset(coordinate, imageType, binding, sizeof(uint32_t));
// TODO: texelOffset is in bytes. get rid of shift once Ben's changes for DivergentPointer land
SIMD::Int texelOffset = GetTexelOffset(coordinate, imageType, binding, sizeof(uint32_t)) >> 2;
auto &dst = state->routine->createIntermediate(resultId, resultType.sizeInComponents);
dst.move(0, texelOffset);
return EmitResult::Continue;
}
SpirvShader::EmitResult SpirvShader::EmitAtomicOp(InsnIterator insn, EmitState *state) const
{
auto &resultType = getType(Type::ID(insn.word(1)));
Object::ID resultId = insn.word(2);
Object::ID semanticsId = insn.word(5);
auto memorySemantics = static_cast<spv::MemorySemanticsMask>(getObject(semanticsId).constantValue[0]);
auto memoryOrder = MemoryOrder(memorySemantics);
auto value = GenericValue(this, state->routine, insn.word(6));
auto &dst = state->routine->createIntermediate(resultId, resultType.sizeInComponents);
auto ptr = Pointer<UInt>(state->routine->getPointer(insn.word(3)));
auto offsets = state->routine->getIntermediate(insn.word(3)).UInt(0);
SIMD::UInt x;
for (int j = 0; j < SIMD::Width; j++)
{
If(Extract(state->activeLaneMask(), j) != 0)
{
auto offset = Extract(offsets, j);
auto laneValue = Extract(value.UInt(0), j);
UInt v;
switch (insn.opcode())
{
case spv::OpAtomicIAdd:
v = AddAtomic(&ptr[offset], laneValue, memoryOrder);
break;
case spv::OpAtomicAnd:
v = AndAtomic(&ptr[offset], laneValue, memoryOrder);
break;
case spv::OpAtomicOr:
v = OrAtomic(&ptr[offset], laneValue, memoryOrder);
break;
case spv::OpAtomicXor:
v = XorAtomic(&ptr[offset], laneValue, memoryOrder);
break;
case spv::OpAtomicSMin:
v = As<UInt>(MinAtomic(As<Pointer<Int>>(&ptr[offset]), As<Int>(laneValue), memoryOrder));
break;
case spv::OpAtomicSMax:
v = As<UInt>(MaxAtomic(As<Pointer<Int>>(&ptr[offset]), As<Int>(laneValue), memoryOrder));
break;
case spv::OpAtomicUMin:
v = MinAtomic(&ptr[offset], laneValue, memoryOrder);
break;
case spv::OpAtomicUMax:
v = MaxAtomic(&ptr[offset], laneValue, memoryOrder);
break;
case spv::OpAtomicExchange:
v = ExchangeAtomic(&ptr[offset], laneValue, memoryOrder);
break;
default:
UNIMPLEMENTED("Atomic op", OpcodeName(insn.opcode()).c_str());
break;
}
x = Insert(x, v, j);
}
}
dst.move(0, x);
return EmitResult::Continue;
}
void SpirvShader::emitEpilog(SpirvRoutine *routine) const
{
for (auto insn : *this)
......
src/Pipeline/SpirvShader.hpp
......@@ -729,6 +729,7 @@ namespace sw
EmitResult EmitImageRead(InsnIterator insn, EmitState *state) const;
EmitResult EmitImageWrite(InsnIterator insn, EmitState *state) const;
EmitResult EmitImageTexelPointer(InsnIterator insn, EmitState *state) const;
EmitResult EmitAtomicOp(InsnIterator insn, EmitState *state) const;
SIMD::Int GetTexelOffset(GenericValue const & coordinate, Type const & imageType, Pointer<Byte> descriptor, int texelSize) const;
......
src/Reactor/LLVMReactor.cpp
......@@ -1451,10 +1451,46 @@ namespace rr
T(llvm::PointerType::get(T(type), 0)));
}
Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
Value *Nucleus::createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Add, V(ptr), V(value), llvm::AtomicOrdering::SequentiallyConsistent));
return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Add, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
Value *Nucleus::createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::And, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
Value *Nucleus::createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Or, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
Value *Nucleus::createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Xor, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
Value *Nucleus::createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Min, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
Value *Nucleus::createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Max, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
Value *Nucleus::createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder)
{
RR_DEBUG_INFO_UPDATE_LOC();
return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
}
Value *Nucleus::createTrunc(Value *v, Type *destType)
......
src/Reactor/Nucleus.hpp
......@@ -101,7 +101,13 @@ namespace rr
static Value *createGEP(Value *ptr, Type *type, Value *index, bool unsignedIndex);
// Atomic instructions
static Value *createAtomicAdd(Value *ptr, Value *value);
static Value *createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
static Value *createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
// Cast/Conversion Operators
static Value *createTrunc(Value *V, Type *destType);
......
src/Reactor/Reactor.cpp
......@@ -2507,6 +2507,51 @@ namespace rr
return RValue<Long>(Nucleus::createAtomicAdd(x.value, y.value));
}
RValue<UInt> AddAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
{
return RValue<UInt>(Nucleus::createAtomicAdd(x.value, y.value, memoryOrder));
}
RValue<UInt> AndAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
{
return RValue<UInt>(Nucleus::createAtomicAnd(x.value, y.value, memoryOrder));
}
RValue<UInt> OrAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
{
return RValue<UInt>(Nucleus::createAtomicOr(x.value, y.value, memoryOrder));
}
RValue<UInt> XorAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
{
return RValue<UInt>(Nucleus::createAtomicXor(x.value, y.value, memoryOrder));
}
RValue<Int> MinAtomic(RValue<Pointer<Int> > x, RValue<Int> y, std::memory_order memoryOrder)
{
return RValue<Int>(Nucleus::createAtomicMin(x.value, y.value, memoryOrder));
}
RValue<UInt> MinAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
{
return RValue<UInt>(Nucleus::createAtomicMin(x.value, y.value, memoryOrder));
}
RValue<Int> MaxAtomic(RValue<Pointer<Int> > x, RValue<Int> y, std::memory_order memoryOrder)
{
return RValue<Int>(Nucleus::createAtomicMax(x.value, y.value, memoryOrder));
}
RValue<UInt> MaxAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
{
return RValue<UInt>(Nucleus::createAtomicMax(x.value, y.value, memoryOrder));
}
RValue<UInt> ExchangeAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, std::memory_order memoryOrder)
{
return RValue<UInt>(Nucleus::createAtomicExchange(x.value, y.value, memoryOrder));
}
UInt::UInt(Argument<UInt> argument)
{
materialize(); // FIXME(b/129757459)
......
src/Reactor/Reactor.hpp
......@@ -1267,6 +1267,17 @@ namespace rr
RValue<UInt> Max(RValue<UInt> x, RValue<UInt> y);
RValue<UInt> Min(RValue<UInt> x, RValue<UInt> y);
RValue<UInt> Clamp(RValue<UInt> x, RValue<UInt> min, RValue<UInt> max);
RValue<UInt> AddAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
RValue<UInt> AndAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
RValue<UInt> OrAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
RValue<UInt> XorAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
RValue<Int> MinAtomic(RValue<Pointer<Int>> x, RValue<Int> y, std::memory_order memoryOrder);
RValue<Int> MaxAtomic(RValue<Pointer<Int>> x, RValue<Int> y, std::memory_order memoryOrder);
RValue<UInt> MinAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
RValue<UInt> MaxAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
RValue<UInt> ExchangeAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
// RValue<UInt> RoundUInt(RValue<Float> cast);
class Int2 : public LValue<Int2>
......
src/Reactor/SubzeroReactor.cpp
......@@ -1033,12 +1033,48 @@ namespace rr
return createAdd(ptr, index);
}
Value *Nucleus::createAtomicAdd(Value *ptr, Value *value)
Value *Nucleus::createAtomicAdd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
UNIMPLEMENTED("createAtomicAdd");
return nullptr;
}
Value *Nucleus::createAtomicAnd(Value *ptr, Value *value, std::memory_order memoryOrder)
{
UNIMPLEMENTED("createAtomicAnd");
return nullptr;
}
Value *Nucleus::createAtomicOr(Value *ptr, Value *value, std::memory_order memoryOrder)
{
UNIMPLEMENTED("createAtomicOr");
return nullptr;
}
Value *Nucleus::createAtomicXor(Value *ptr, Value *value, std::memory_order memoryOrder)
{
UNIMPLEMENTED("createAtomicXor");
return nullptr;
}
Value *Nucleus::createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder)
{
UNIMPLEMENTED("createAtomicMin");
return nullptr;
}
Value *Nucleus::createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder)
{
UNIMPLEMENTED("createAtomicMax");
return nullptr;
}
Value *Nucleus::createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder)
{
UNIMPLEMENTED("createAtomicExchange");
return nullptr;
}
static Value *createCast(Ice::InstCast::OpKind op, Value *v, Type *destType)
{
if(v->getType() == T(destType))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment