Add support for OpAtomicCompareExchange

src/Pipeline/SpirvShader.cpp

@@ -687,6 +687,7 @@ namespace sw
 			case spv::OpAtomicOr:
 			case spv::OpAtomicXor:
 			case spv::OpAtomicExchange:
+			case spv::OpAtomicCompareExchange:
 			case spv::OpPhi:
 			case spv::OpImageSampleImplicitLod:
 			case spv::OpImageQuerySize:
@@ -1998,6 +1999,9 @@ namespace sw
 		case spv::OpAtomicExchange:
 			return EmitAtomicOp(insn, state);
 
+		case spv::OpAtomicCompareExchange:
+			return EmitAtomicCompareExchange(insn, state);
+
 		case spv::OpAccessChain:
 		case spv::OpInBoundsAccessChain:
 			return EmitAccessChain(insn, state);
@@ -4784,6 +4788,40 @@ namespace sw
 		return EmitResult::Continue;
 	}
 
+	SpirvShader::EmitResult SpirvShader::EmitAtomicCompareExchange(InsnIterator insn, EmitState *state) const
+	{
+		// Separate from EmitAtomicOp due to different instruction encoding
+		auto &resultType = getType(Type::ID(insn.word(1)));
+		Object::ID resultId = insn.word(2);
+
+		auto memorySemanticsEqual = static_cast<spv::MemorySemanticsMask>(getObject(insn.word(5)).constantValue[0]);
+		auto memoryOrderEqual = MemoryOrder(memorySemanticsEqual);
+		auto memorySemanticsUnequal = static_cast<spv::MemorySemanticsMask>(getObject(insn.word(6)).constantValue[0]);
+		auto memoryOrderUnequal = MemoryOrder(memorySemanticsUnequal);
+
+		auto value = GenericValue(this, state->routine, insn.word(7));
+		auto comparator = GenericValue(this, state->routine, insn.word(8));
+		auto &dst = state->routine->createIntermediate(resultId, resultType.sizeInComponents);
+		auto ptr = Pointer<UInt>(state->routine->getPointer(insn.word(3)));
+		auto offsets = state->routine->getIntermediate(insn.word(3)).UInt(0);
+
+		SIMD::UInt x;
+		for (int j = 0; j < SIMD::Width; j++)
+		{
+			If(Extract(state->activeLaneMask(), j) != 0)
+			{
+				auto offset = Extract(offsets, j);
+				auto laneValue = Extract(value.UInt(0), j);
+				auto laneComparator = Extract(comparator.UInt(0), j);
+				UInt v = CompareExchangeAtomic(&ptr[offset], laneValue, laneComparator, memoryOrderEqual, memoryOrderUnequal);
+				x = Insert(x, v, j);
+			}
+		}
+
+		dst.move(0, x);
+		return EmitResult::Continue;
+	}
+
 	void SpirvShader::emitEpilog(SpirvRoutine *routine) const
 	{
 		for (auto insn : *this)

src/Pipeline/SpirvShader.hpp

@@ -730,6 +730,7 @@ namespace sw
 		EmitResult EmitImageWrite(InsnIterator insn, EmitState *state) const;
 		EmitResult EmitImageTexelPointer(InsnIterator insn, EmitState *state) const;
 		EmitResult EmitAtomicOp(InsnIterator insn, EmitState *state) const;
+		EmitResult EmitAtomicCompareExchange(InsnIterator insn, EmitState *state) const;
 
 		SIMD::Int GetTexelOffset(GenericValue const & coordinate, Type const & imageType, Pointer<Byte> descriptor, int texelSize) const;
 

src/Reactor/LLVMReactor.cpp

@@ -1493,6 +1493,15 @@ namespace rr
 		return V(::builder->CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, V(ptr), V(value), atomicOrdering(true, memoryOrder)));
 	}
 
+	Value *Nucleus::createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal)
+	{
+		RR_DEBUG_INFO_UPDATE_LOC();
+		// Note: AtomicCmpXchgInstruction returns a 1-member struct containing the result, not the result directly.
+		return V(::builder->CreateExtractValue(
+				::builder->CreateAtomicCmpXchg(V(ptr), V(compare), V(value), atomicOrdering(true, memoryOrderEqual), atomicOrdering(true, memoryOrderUnequal)),
+				llvm::ArrayRef<unsigned>(0u)));
+	}
+
 	Value *Nucleus::createTrunc(Value *v, Type *destType)
 	{
 		RR_DEBUG_INFO_UPDATE_LOC();

src/Reactor/Nucleus.hpp

@@ -108,6 +108,7 @@ namespace rr
 		static Value *createAtomicMin(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
 		static Value *createAtomicMax(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
 		static Value *createAtomicExchange(Value *ptr, Value *value, std::memory_order memoryOrder = std::memory_order_relaxed);
+		static Value *createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal);
 
 		// Cast/Conversion Operators
 		static Value *createTrunc(Value *V, Type *destType);

src/Reactor/Reactor.cpp

@@ -2552,6 +2552,11 @@ namespace rr
 		return RValue<UInt>(Nucleus::createAtomicExchange(x.value, y.value, memoryOrder));
 	}
 
+	RValue<UInt> CompareExchangeAtomic(RValue<Pointer<UInt> > x, RValue<UInt> y, RValue<UInt> compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal)
+	{
+		return RValue<UInt>(Nucleus::createAtomicCompareExchange(x.value, y.value, compare.value, memoryOrderEqual, memoryOrderUnequal));
+	}
+
 	UInt::UInt(Argument<UInt> argument)
 	{
 		materialize();  // FIXME(b/129757459)

src/Reactor/Reactor.hpp

@@ -1277,6 +1277,7 @@ namespace rr
 	RValue<UInt> MinAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
 	RValue<UInt> MaxAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
 	RValue<UInt> ExchangeAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, std::memory_order memoryOrder);
+	RValue<UInt> CompareExchangeAtomic(RValue<Pointer<UInt>> x, RValue<UInt> y, RValue<UInt> compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal);
 
 //	RValue<UInt> RoundUInt(RValue<Float> cast);
 

src/Reactor/SubzeroReactor.cpp

@@ -1075,6 +1075,12 @@ namespace rr
 		return nullptr;
 	}
 
+	Value *Nucleus::createAtomicCompareExchange(Value *ptr, Value *value, Value *compare, std::memory_order memoryOrderEqual, std::memory_order memoryOrderUnequal)
+	{
+		UNIMPLEMENTED("createAtomicCompareExchange");
+		return nullptr;
+	}
+
 	static Value *createCast(Ice::InstCast::OpKind op, Value *v, Type *destType)
 	{
 		if(v->getType() == T(destType))