Skip to content

S
swiftshader
Commit 71f6c8de by Jaydeep Patil Committed by Jim Stichnoth
Options

[SubZero] Generate spin-lock for atomic load store

The patch generates LL-SC sequence for atomic load store operations. 64-bit atomic operations are lowered to __sync_*_8 functions. R=stichnot@chromium.org Patch from Jaydeep Patil <jaydeep.patil@imgtec.com>. Review-Url: https://codereview.chromium.org/2682673002 .
parent c48bb8b0
Showing with 295 additions and 318 deletions
runtime/szrt.c
......@@ -64,3 +64,11 @@ double __Sz_sitofp_i64_f64(int64_t Value) { return (double)Value; }
// unsandboxed_irt:
// __nacl_read_tp
// __aeabi_read_tp [arm32 only]
// MIPS runtime library:
// __sync_fetch_and_add_8
// __sync_fetch_and_and_8
// __sync_fetch_and_or_8
// __sync_fetch_and_sub_8
// __sync_fetch_and_xor_8
// __sync_lock_test_and_set_8
// __sync_val_compare_and_swap_8
src/IceTargetLoweringMIPS32.cpp
......@@ -620,6 +620,128 @@ void TargetMIPS32::genTargetHelperCallFor(Inst *Instr) {
switch (ID) {
default:
return;
case Intrinsics::AtomicLoad: {
if (DestTy != IceType_i64)
return;
if (!Intrinsics::isMemoryOrderValid(
ID, getConstantMemoryOrder(IntrinsicCall->getArg(1)))) {
Func->setError("Unexpected memory ordering for AtomicLoad");
return;
}
Operand *Addr = IntrinsicCall->getArg(0);
Operand *TargetHelper = Ctx->getConstantExternSym(
Ctx->getGlobalString("__sync_val_compare_and_swap_8"));
static constexpr SizeT MaxArgs = 3;
auto *_0 = Ctx->getConstantZero(IceType_i64);
auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper,
NoTailCall, IsTargetHelperCall);
Call->addArg(Addr);
Call->addArg(_0);
Call->addArg(_0);
Context.insert<InstMIPS32Sync>();
Instr->setDeleted();
return;
}
case Intrinsics::AtomicStore: {
Operand *Val = IntrinsicCall->getArg(0);
if (Val->getType() != IceType_i64)
return;
if (!Intrinsics::isMemoryOrderValid(
ID, getConstantMemoryOrder(IntrinsicCall->getArg(2)))) {
Func->setError("Unexpected memory ordering for AtomicStore");
return;
}
Operand *Addr = IntrinsicCall->getArg(1);
Variable *NoDest = nullptr;
Operand *TargetHelper = Ctx->getConstantExternSym(
Ctx->getGlobalString("__sync_lock_test_and_set_8"));
Context.insert<InstMIPS32Sync>();
static constexpr SizeT MaxArgs = 2;
auto *Call = Context.insert<InstCall>(MaxArgs, NoDest, TargetHelper,
NoTailCall, IsTargetHelperCall);
Call->addArg(Addr);
Call->addArg(Val);
Context.insert<InstMIPS32Sync>();
Instr->setDeleted();
return;
}
case Intrinsics::AtomicCmpxchg: {
if (DestTy != IceType_i64)
return;
if (!Intrinsics::isMemoryOrderValid(
ID, getConstantMemoryOrder(IntrinsicCall->getArg(3)),
getConstantMemoryOrder(IntrinsicCall->getArg(4)))) {
Func->setError("Unexpected memory ordering for AtomicCmpxchg");
return;
}
Operand *Addr = IntrinsicCall->getArg(0);
Operand *Oldval = IntrinsicCall->getArg(1);
Operand *Newval = IntrinsicCall->getArg(2);
Operand *TargetHelper = Ctx->getConstantExternSym(
Ctx->getGlobalString("__sync_val_compare_and_swap_8"));
Context.insert<InstMIPS32Sync>();
static constexpr SizeT MaxArgs = 3;
auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper,
NoTailCall, IsTargetHelperCall);
Call->addArg(Addr);
Call->addArg(Oldval);
Call->addArg(Newval);
Context.insert<InstMIPS32Sync>();
Instr->setDeleted();
return;
}
case Intrinsics::AtomicRMW: {
if (DestTy != IceType_i64)
return;
if (!Intrinsics::isMemoryOrderValid(
ID, getConstantMemoryOrder(IntrinsicCall->getArg(3)))) {
Func->setError("Unexpected memory ordering for AtomicRMW");
return;
}
auto Operation = static_cast<Intrinsics::AtomicRMWOperation>(
llvm::cast<ConstantInteger32>(IntrinsicCall->getArg(0))->getValue());
auto *Addr = IntrinsicCall->getArg(1);
auto *Newval = IntrinsicCall->getArg(2);
Operand *TargetHelper;
switch (Operation) {
case Intrinsics::AtomicAdd:
TargetHelper = Ctx->getConstantExternSym(
Ctx->getGlobalString("__sync_fetch_and_add_8"));
break;
case Intrinsics::AtomicSub:
TargetHelper = Ctx->getConstantExternSym(
Ctx->getGlobalString("__sync_fetch_and_sub_8"));
break;
case Intrinsics::AtomicOr:
TargetHelper = Ctx->getConstantExternSym(
Ctx->getGlobalString("__sync_fetch_and_or_8"));
break;
case Intrinsics::AtomicAnd:
TargetHelper = Ctx->getConstantExternSym(
Ctx->getGlobalString("__sync_fetch_and_and_8"));
break;
case Intrinsics::AtomicXor:
TargetHelper = Ctx->getConstantExternSym(
Ctx->getGlobalString("__sync_fetch_and_xor_8"));
break;
case Intrinsics::AtomicExchange:
TargetHelper = Ctx->getConstantExternSym(
Ctx->getGlobalString("__sync_lock_test_and_set_8"));
break;
default:
llvm::report_fatal_error("Unknown AtomicRMW operation");
return;
}
Context.insert<InstMIPS32Sync>();
static constexpr SizeT MaxArgs = 2;
auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper,
NoTailCall, IsTargetHelperCall);
Call->addArg(Addr);
Call->addArg(Newval);
Context.insert<InstMIPS32Sync>();
Instr->setDeleted();
return;
}
case Intrinsics::Ctpop: {
Operand *Src0 = IntrinsicCall->getArg(0);
Operand *TargetHelper =
......@@ -4455,34 +4577,66 @@ void TargetMIPS32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
return;
}
if (DestTy == IceType_i64) {
llvm::report_fatal_error("AtomicLoad.i64 should have been prelowered.");
return;
} else if (DestTy == IceType_i32) {
auto *T1 = makeReg(DestTy);
auto *RegAt = getPhysicalRegister(RegMIPS32::Reg_AT);
auto *Base = legalizeToReg(Instr->getArg(0));
auto *AddrLo = OperandMIPS32Mem::create(
Func, IceType_i32, Base,
llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0)));
auto *AddrHi = OperandMIPS32Mem::create(
Func, IceType_i32, Base,
llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(4)));
auto *T_Lo = makeReg(IceType_i32);
auto *T_Hi = makeReg(IceType_i32);
auto *Dest64 = llvm::cast<Variable64On32>(Dest);
lowerLoad(InstLoad::create(Func, T_Lo, AddrLo, IceType_i32));
lowerLoad(InstLoad::create(Func, T_Hi, AddrHi, IceType_i32));
auto *Addr = formMemoryOperand(Base, DestTy);
InstMIPS32Label *Retry = InstMIPS32Label::create(Func, this);
InstMIPS32Label *Exit = InstMIPS32Label::create(Func, this);
constexpr CfgNode *NoTarget = nullptr;
_sync();
_mov(Dest64->getLo(), T_Lo);
_mov(Dest64->getHi(), T_Hi);
// Adding a fake-use of T to ensure the atomic load is not removed if Dest
// is unused.
Context.insert<InstFakeUse>(T_Lo);
Context.insert<InstFakeUse>(T_Hi);
Context.insert(Retry);
Sandboxer(this).ll(T1, Addr);
_br(NoTarget, NoTarget, T1, getZero(), Exit, CondMIPS32::Cond::NE);
_addiu(RegAt, getZero(), 0); // Loaded value is zero here, writeback zero
Sandboxer(this).sc(RegAt, Addr);
_br(NoTarget, NoTarget, RegAt, getZero(), Retry, CondMIPS32::Cond::EQ);
Context.insert(Exit);
_sync();
_mov(Dest, T1);
Context.insert<InstFakeUse>(T1);
} else {
auto *T = makeReg(DestTy);
const uint32_t Mask = (1 << (CHAR_BITS * typeWidthInBytes(DestTy))) - 1;
auto *Base = legalizeToReg(Instr->getArg(0));
lowerLoad(InstLoad::create(Func, T, formMemoryOperand(Base, DestTy)));
auto *T1 = makeReg(IceType_i32);
auto *T2 = makeReg(IceType_i32);
auto *T3 = makeReg(IceType_i32);
auto *T4 = makeReg(IceType_i32);
auto *T5 = makeReg(IceType_i32);
auto *T6 = makeReg(IceType_i32);
auto *SrcMask = makeReg(IceType_i32);
auto *Tdest = makeReg(IceType_i32);
auto *RegAt = getPhysicalRegister(RegMIPS32::Reg_AT);
InstMIPS32Label *Retry = InstMIPS32Label::create(Func, this);
InstMIPS32Label *Exit = InstMIPS32Label::create(Func, this);
constexpr CfgNode *NoTarget = nullptr;
_sync();
_mov(Dest, T);
// Adding a fake-use of T to ensure the atomic load is not removed if Dest
// is unused.
Context.insert<InstFakeUse>(T);
_addiu(T1, getZero(), -4); // Address mask 0xFFFFFFFC
_andi(T2, Base, 3); // Last two bits of the address
_and(T3, Base, T1); // Align the address
_sll(T4, T2, 3);
_ori(T5, getZero(), Mask);
_sllv(SrcMask, T5, T4); // Source mask
auto *Addr = formMemoryOperand(T3, IceType_i32);
Context.insert(Retry);
Sandboxer(this).ll(T6, Addr);
_and(Tdest, T6, SrcMask);
_br(NoTarget, NoTarget, T6, getZero(), Exit, CondMIPS32::Cond::NE);
_addiu(RegAt, getZero(), 0); // Loaded value is zero here, writeback zero
Sandboxer(this).sc(RegAt, Addr);
_br(NoTarget, NoTarget, RegAt, getZero(), Retry, CondMIPS32::Cond::EQ);
Context.insert(Exit);
auto *T7 = makeReg(IceType_i32);
auto *T8 = makeReg(IceType_i32);
_srlv(T7, Tdest, T4);
_andi(T8, T7, Mask);
_sync();
_mov(Dest, T8);
Context.insert<InstFakeUse>(T6);
Context.insert<InstFakeUse>(SrcMask);
}
return;
}
......@@ -4497,35 +4651,59 @@ void TargetMIPS32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
auto *Val = Instr->getArg(0);
auto Ty = Val->getType();
if (Ty == IceType_i64) {
Variable *ValHi, *ValLo;
_sync();
if (auto *C64 = llvm::dyn_cast<ConstantInteger64>(Val)) {
const uint64_t Value = C64->getValue();
uint64_t Upper32Bits = (Value >> INT32_BITS) & 0xFFFFFFFF;
uint64_t Lower32Bits = Value & 0xFFFFFFFF;
ValLo = legalizeToReg(Ctx->getConstantInt32(Lower32Bits));
ValHi = legalizeToReg(Ctx->getConstantInt32(Upper32Bits));
} else {
auto *Val64 = llvm::cast<Variable64On32>(Val);
ValLo = legalizeToReg(loOperand(Val64));
ValHi = legalizeToReg(hiOperand(Val64));
}
llvm::report_fatal_error("AtomicStore.i64 should have been prelowered.");
return;
} else if (Ty == IceType_i32) {
auto *Val = legalizeToReg(Instr->getArg(0));
auto *Base = legalizeToReg(Instr->getArg(1));
auto *AddrLo = OperandMIPS32Mem::create(
Func, IceType_i32, Base,
llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0)));
auto *AddrHi = OperandMIPS32Mem::create(
Func, IceType_i32, Base,
llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(4)));
lowerStore(InstStore::create(Func, ValLo, AddrLo, IceType_i32));
lowerStore(InstStore::create(Func, ValHi, AddrHi, IceType_i32));
auto *Addr = formMemoryOperand(Base, Ty);
InstMIPS32Label *Retry = InstMIPS32Label::create(Func, this);
constexpr CfgNode *NoTarget = nullptr;
auto *T1 = makeReg(IceType_i32);
auto *RegAt = getPhysicalRegister(RegMIPS32::Reg_AT);
_sync();
} else {
Context.insert(Retry);
Sandboxer(this).ll(T1, Addr);
_mov(RegAt, Val);
Sandboxer(this).sc(RegAt, Addr);
_br(NoTarget, NoTarget, RegAt, getZero(), Retry, CondMIPS32::Cond::EQ);
Context.insert<InstFakeUse>(T1); // To keep LL alive
_sync();
} else {
auto *Val = legalizeToReg(Instr->getArg(0));
auto *Base = legalizeToReg(Instr->getArg(1));
lowerStore(InstStore::create(Func, Val, formMemoryOperand(Base, DestTy)));
InstMIPS32Label *Retry = InstMIPS32Label::create(Func, this);
constexpr CfgNode *NoTarget = nullptr;
auto *T1 = makeReg(IceType_i32);
auto *T2 = makeReg(IceType_i32);
auto *T3 = makeReg(IceType_i32);
auto *T4 = makeReg(IceType_i32);
auto *T5 = makeReg(IceType_i32);
auto *T6 = makeReg(IceType_i32);
auto *T7 = makeReg(IceType_i32);
auto *RegAt = getPhysicalRegister(RegMIPS32::Reg_AT);
auto *SrcMask = makeReg(IceType_i32);
auto *DstMask = makeReg(IceType_i32);
const uint32_t Mask = (1 << (CHAR_BITS * typeWidthInBytes(Ty))) - 1;
_sync();
_addiu(T1, getZero(), -4);
_and(T7, Base, T1);
auto *Addr = formMemoryOperand(T7, Ty);
_andi(T2, Base, 3);
_sll(T3, T2, 3);
_ori(T4, getZero(), Mask);
_sllv(T5, T4, T3);
_sllv(T6, Val, T3);
_nor(SrcMask, getZero(), T5);
_and(DstMask, T6, T5);
Context.insert(Retry);
Sandboxer(this).ll(RegAt, Addr);
_and(RegAt, RegAt, SrcMask);
_or(RegAt, RegAt, DstMask);
Sandboxer(this).sc(RegAt, Addr);
_br(NoTarget, NoTarget, RegAt, getZero(), Retry, CondMIPS32::Cond::EQ);
Context.insert<InstFakeUse>(SrcMask);
Context.insert<InstFakeUse>(DstMask);
_sync();
}
return;
......@@ -4549,73 +4727,9 @@ void TargetMIPS32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
auto *ActualAddress = Instr->getArg(0);
if (DestTy == IceType_i64) {
InstMIPS32Label *Retry1 = InstMIPS32Label::create(Func, this);
auto *T1 = I32Reg();
auto *T2 = I32Reg();
auto *T3 = I32Reg();
auto *T4 = I32Reg();
auto *T5 = I32Reg();
auto *T6 = I32Reg();
Variable *ValHi, *ValLo, *ExpectedLo, *ExpectedHi;
if (llvm::isa<ConstantUndef>(Expected)) {
ExpectedLo = legalizeToReg(Ctx->getConstantZero(IceType_i32));
ExpectedHi = legalizeToReg(Ctx->getConstantZero(IceType_i32));
} else if (auto *Expected64 = llvm::dyn_cast<Variable64On32>(Expected)) {
ExpectedLo = legalizeToReg(loOperand(Expected64));
ExpectedHi = legalizeToReg(hiOperand(Expected64));
} else if (auto *C64 = llvm::dyn_cast<ConstantInteger64>(Expected)) {
const uint64_t Value = C64->getValue();
uint64_t Upper32Bits = (Value >> INT32_BITS) & 0xFFFFFFFF;
uint64_t Lower32Bits = Value & 0xFFFFFFFF;
ExpectedLo = legalizeToReg(Ctx->getConstantInt32(Lower32Bits));
ExpectedHi = legalizeToReg(Ctx->getConstantInt32(Upper32Bits));
} else {
llvm::report_fatal_error(
"AtomicCmpxchg: getArg(1) is nor Constant neither Variable64On32");
}
if (auto *C64 = llvm::dyn_cast<ConstantInteger64>(New)) {
const uint64_t Value = C64->getValue();
uint64_t Upper32Bits = (Value >> INT32_BITS) & 0xFFFFFFFF;
uint64_t Lower32Bits = Value & 0xFFFFFFFF;
ValLo = legalizeToReg(Ctx->getConstantInt32(Lower32Bits));
ValHi = legalizeToReg(Ctx->getConstantInt32(Upper32Bits));
} else {
auto *Val = llvm::cast<Variable64On32>(New);
ValLo = legalizeToReg(loOperand(Val));
ValHi = legalizeToReg(hiOperand(Val));
}
auto *Dest64 = llvm::cast<Variable64On32>(Dest);
auto *BaseR = legalizeToReg(ActualAddress);
auto *AddrLo = OperandMIPS32Mem::create(
Func, IceType_i32, BaseR,
llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0)));
auto *AddrHi = OperandMIPS32Mem::create(
Func, IceType_i32, BaseR,
llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(4)));
lowerLoad(InstLoad::create(Func, T3, AddrLo));
lowerLoad(InstLoad::create(Func, T4, AddrHi));
_sync();
Context.insert(Retry);
Sandboxer(this).ll(T1, AddrLo);
_br(NoTarget, NoTarget, T1, ExpectedLo, Exit, CondMIPS32::Cond::NE);
_mov(T5, ValLo);
Sandboxer(this).sc(T5, AddrLo);
Context.insert<InstFakeUse>(ValLo);
Context.insert<InstFakeUse>(ExpectedLo);
_br(NoTarget, NoTarget, T5, getZero(), Retry, CondMIPS32::Cond::EQ);
Context.insert(Retry1);
Sandboxer(this).ll(T2, AddrHi);
_br(NoTarget, NoTarget, T2, ExpectedHi, Exit, CondMIPS32::Cond::NE);
_mov(T6, ValHi);
Sandboxer(this).sc(T6, AddrHi);
Context.insert<InstFakeUse>(ValHi);
Context.insert<InstFakeUse>(ExpectedHi);
_br(NoTarget, NoTarget, T6, getZero(), Retry1, CondMIPS32::Cond::EQ);
Context.insert<InstFakeUse>(getZero());
Context.insert(Exit);
_mov(Dest64->getLo(), T3);
_mov(Dest64->getHi(), T4);
_sync();
llvm::report_fatal_error(
"AtomicCmpxchg.i64 should have been prelowered.");
return;
} else if (DestTy == IceType_i8 || DestTy == IceType_i16) {
auto *NewR = legalizeToReg(New);
auto *ExpectedR = legalizeToReg(Expected);
......@@ -4635,6 +4749,7 @@ void TargetMIPS32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
auto *T9 = I32Reg();
_addiu(RegAt, getZero(), -4);
_and(T1, ActualAddressR, RegAt);
auto *Addr = formMemoryOperand(T1, DestTy);
_andi(RegAt, ActualAddressR, 3);
_sll(T2, RegAt, 3);
_ori(RegAt, getZero(), Mask);
......@@ -4646,13 +4761,12 @@ void TargetMIPS32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
_sllv(T6, RegAt, T2);
_sync();
Context.insert(Retry);
Sandboxer(this).ll(T7, formMemoryOperand(T1, DestTy));
Sandboxer(this).ll(T7, Addr);
_and(T8, T7, T3);
_br(NoTarget, NoTarget, T8, T5, Exit, CondMIPS32::Cond::NE);
_and(RegAt, T7, T4);
_or(T9, RegAt, T6);
Sandboxer(this).sc(T9, formMemoryOperand(T1, DestTy));
Context.insert<InstFakeUse>(T6);
Sandboxer(this).sc(T9, Addr);
_br(NoTarget, NoTarget, getZero(), T9, Retry, CondMIPS32::Cond::EQ);
Context.insert<InstFakeUse>(getZero());
Context.insert(Exit);
......@@ -4661,6 +4775,11 @@ void TargetMIPS32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
_sra(RegAt, RegAt, ShiftAmount);
_mov(Dest, RegAt);
_sync();
Context.insert<InstFakeUse>(T3);
Context.insert<InstFakeUse>(T4);
Context.insert<InstFakeUse>(T5);
Context.insert<InstFakeUse>(T6);
Context.insert<InstFakeUse>(T8);
Context.insert<InstFakeUse>(ExpectedR);
Context.insert<InstFakeUse>(NewR);
} else {
......@@ -4703,69 +4822,8 @@ void TargetMIPS32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
auto *ActualAddress = Instr->getArg(1);
if (DestTy == IceType_i64) {
InstMIPS32Label *Retry1 = InstMIPS32Label::create(Func, this);
_sync();
Variable *ValHi, *ValLo;
if (auto *C64 = llvm::dyn_cast<ConstantInteger64>(New)) {
const uint64_t Value = C64->getValue();
uint64_t Upper32Bits = (Value >> INT32_BITS) & 0xFFFFFFFF;
uint64_t Lower32Bits = Value & 0xFFFFFFFF;
ValLo = legalizeToReg(Ctx->getConstantInt32(Lower32Bits));
ValHi = legalizeToReg(Ctx->getConstantInt32(Upper32Bits));
} else {
auto *Val = llvm::cast<Variable64On32>(New);
ValLo = legalizeToReg(loOperand(Val));
ValHi = legalizeToReg(hiOperand(Val));
}
auto *Dest64 = llvm::cast<Variable64On32>(Dest);
auto *BaseR = legalizeToReg(ActualAddress);
auto *AddrLo = OperandMIPS32Mem::create(
Func, IceType_i32, BaseR,
llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0)));
auto *AddrHi = OperandMIPS32Mem::create(
Func, IceType_i32, BaseR,
llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(4)));
auto *RegAt = getPhysicalRegister(RegMIPS32::Reg_AT);
auto *T1 = I32Reg();
auto *T2 = I32Reg();
auto *T3 = I32Reg();
Context.insert(Retry);
Sandboxer(this).ll(T1, AddrLo);
if (Operation == Intrinsics::AtomicExchange) {
_mov(RegAt, ValLo);
Context.insert<InstFakeUse>(T1);
} else if (Operation == Intrinsics::AtomicAdd) {
createArithInst(Operation, RegAt, T1, ValLo);
_sltu(T2, RegAt, T1);
} else if (Operation == Intrinsics::AtomicSub) {
createArithInst(Operation, RegAt, T1, ValLo);
_sltu(T2, T1, ValLo);
} else {
createArithInst(Operation, RegAt, T1, ValLo);
}
Sandboxer(this).sc(RegAt, AddrLo);
_br(NoTarget, NoTarget, RegAt, getZero(), Retry, CondMIPS32::Cond::EQ);
Context.insert<InstFakeUse>(getZero());
_mov(Dest64->getLo(), T1);
Context.insert(Retry1);
Sandboxer(this).ll(T3, AddrHi);
if (Operation == Intrinsics::AtomicAdd ||
Operation == Intrinsics::AtomicSub) {
_addu(RegAt, T2, ValHi);
createArithInst(Operation, RegAt, T3, RegAt);
} else if (Operation == Intrinsics::AtomicExchange) {
_mov(RegAt, ValHi);
Context.insert<InstFakeUse>(T3);
} else {
createArithInst(Operation, RegAt, T3, ValHi);
}
Sandboxer(this).sc(RegAt, AddrHi);
_br(NoTarget, NoTarget, RegAt, getZero(), Retry1, CondMIPS32::Cond::EQ);
Context.insert<InstFakeUse>(getZero());
_mov(Dest64->getHi(), T3);
Context.insert<InstFakeUse>(ValLo);
Context.insert<InstFakeUse>(ValHi);
_sync();
llvm::report_fatal_error("AtomicRMW.i64 should have been prelowered.");
return;
} else if (DestTy == IceType_i8 || DestTy == IceType_i16) {
const uint32_t ShiftAmount =
INT32_BITS - (CHAR_BITS * typeWidthInBytes(DestTy));
......@@ -5624,6 +5682,14 @@ void TargetMIPS32::lowerUnreachable(const InstUnreachable *) {
_teq(getZero(), getZero(), TrapCodeZero);
}
void TargetMIPS32::lowerOther(const Inst *Instr) {
if (llvm::isa<InstMIPS32Sync>(Instr)) {
_sync();
} else {
TargetLowering::lowerOther(Instr);
}
}
// Turn an i64 Phi instruction into a pair of i32 Phi instructions, to preserve
// integrity of liveness analysis. Undef values are also turned into zeroes,
// since loOperand() and hiOperand() don't expect Undef input.
......
src/IceTargetLoweringMIPS32.h
......@@ -787,6 +787,7 @@ protected:
void lowerStore(const InstStore *Instr) override;
void lowerSwitch(const InstSwitch *Instr) override;
void lowerUnreachable(const InstUnreachable *Instr) override;
void lowerOther(const Inst *Instr) override;
void prelowerPhis() override;
uint32_t getCallStackArgumentsSizeBytes(const InstCall *Instr) override;
void genTargetHelperCallFor(Inst *Instr) override;
......
tests_lit/llvm2ice_tests/nacl-atomic-intrinsics.ll
......@@ -89,7 +89,9 @@ entry:
; ARM32: ldrb r{{[0-9]+}}, [r{{[0-9]+}}
; ARM32: dmb
; MIPS32-LABEL: test_atomic_load_8
; MIPS32: lb
; MIPS32: sync
; MIPS32: ll
; MIPS32: sc
; MIPS32: sync
define internal i32 @test_atomic_load_16(i32 %iptr) {
......@@ -107,7 +109,9 @@ entry:
; ARM32: ldrh r{{[0-9]+}}, [r{{[0-9]+}}
; ARM32: dmb
; MIPS32-LABEL: test_atomic_load_16
; MIPS32: lh
; MIPS32: sync
; MIPS32: ll
; MIPS32: sc
; MIPS32: sync
define internal i32 @test_atomic_load_32(i32 %iptr) {
......@@ -123,7 +127,9 @@ entry:
; ARM32: ldr r{{[0-9]+}}, [r{{[0-9]+}}
; ARM32: dmb
; MIPS32-LABEL: test_atomic_load_32
; MIPS32: lw
; MIPS32: sync
; MIPS32: ll
; MIPS32: sc
; MIPS32: sync
define internal i64 @test_atomic_load_64(i32 %iptr) {
......@@ -139,8 +145,7 @@ entry:
; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}
; ARM32: dmb
; MIPS32-LABEL: test_atomic_load_64
; MIPS32: lw
; MIPS32: lw
; MIPS32: jal __sync_val_compare_and_swap_8
; MIPS32: sync
define internal i32 @test_atomic_load_32_with_arith(i32 %iptr) {
......@@ -165,8 +170,11 @@ next:
; ARM32: ldr r{{[0-9]+}}, [r{{[0-9]+}}
; ARM32: dmb
; MIPS32-LABEL: test_atomic_load_32_with_arith
; MIPS32: lw
; MIPS32: sync
; MIPS32: ll
; MIPS32: sc
; MIPS32: sync
; MIPS32: subu
define internal i32 @test_atomic_load_32_ignored(i32 %iptr) {
entry:
......@@ -184,7 +192,9 @@ entry:
; ARM32: ldr r{{[0-9]+}}, [r{{[0-9]+}}
; ARM32: dmb
; MIPS32-LABEL: test_atomic_load_32_ignored
; MIPS32: lw
; MIPS32: sync
; MIPS32: ll
; MIPS32: sc
; MIPS32: sync
define internal i64 @test_atomic_load_64_ignored(i32 %iptr) {
......@@ -200,8 +210,7 @@ entry:
; ARM32: ldrexd r{{[0-9]+}}, r{{[0-9]+}}, [r{{[0-9]+}}
; ARM32: dmb
; MIPS32-LABEL: test_atomic_load_64_ignored
; MIPS32: lw
; MIPS32: lw
; MIPS32: jal __sync_val_compare_and_swap_8
; MIPS32: sync
;;; Store
......@@ -222,7 +231,8 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_store_8
; MIPS32: sync
; MIPS32: sb
; MIPS32: ll
; MIPS32: sc
; MIPS32: sync
define internal void @test_atomic_store_16(i32 %iptr, i32 %v) {
......@@ -241,7 +251,8 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_store_16
; MIPS32: sync
; MIPS32: sh
; MIPS32: ll
; MIPS32: sc
; MIPS32: sync
define internal void @test_atomic_store_32(i32 %iptr, i32 %v) {
......@@ -259,7 +270,8 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_store_32
; MIPS32: sync
; MIPS32: sw
; MIPS32: ll
; MIPS32: sc
; MIPS32: sync
define internal void @test_atomic_store_64(i32 %iptr, i64 %v) {
......@@ -281,8 +293,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_store_64
; MIPS32: sync
; MIPS32: sw
; MIPS32: sw
; MIPS32: jal __sync_lock_test_and_set_8
; MIPS32: sync
define internal void @test_atomic_store_64_const(i32 %iptr) {
......@@ -313,8 +324,7 @@ entry:
; MIPS32: lui {{.*}}, 29646
; MIPS32: ori {{.*}},{{.*}}, 12274
; MIPS32: addiu {{.*}}, $zero, 2874
; MIPS32: sw
; MIPS32: sw
; MIPS32: jal __sync_lock_test_and_set_8
; MIPS32: sync
;;; RMW
......@@ -459,16 +469,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_rmw_add_64
; MIPS32: sync
; MIPS32: ll
; MIPS32: addu
; MIPS32: sltu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: ll
; MIPS32: addu
; MIPS32: addu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: jal __sync_fetch_and_add_8
; MIPS32: sync
; Same test as above, but with a global address to test FakeUse issues.
......@@ -489,16 +490,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_rmw_add_64_global
; MIPS32: sync
; MIPS32: ll
; MIPS32: addu
; MIPS32: sltu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: ll
; MIPS32: addu
; MIPS32: addu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: jal __sync_fetch_and_add_8
; MIPS32: sync
; Test with some more register pressure. When we have an alloca, ebp is
......@@ -543,16 +535,7 @@ eblock:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_rmw_add_64_alloca
; MIPS32: sync
; MIPS32: ll
; MIPS32: addu
; MIPS32: sltu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: ll
; MIPS32: addu
; MIPS32: addu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: jal __sync_fetch_and_add_8
; MIPS32: sync
define internal i32 @test_atomic_rmw_add_32_ignored(i32 %iptr, i32 %v) {
......@@ -621,16 +604,7 @@ err:
; ARM32: b
; MIPS32-LABEL: test_atomic_rmw_add_64_loop
; MIPS32: sync
; MIPS32: ll
; MIPS32: addu
; MIPS32: sltu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: ll
; MIPS32: addu
; MIPS32: addu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: jal __sync_fetch_and_add_8
; MIPS32: sync
;; sub
......@@ -770,16 +744,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_rmw_sub_64
; MIPS32: sync
; MIPS32: ll
; MIPS32: subu
; MIPS32: sltu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: ll
; MIPS32: addu
; MIPS32: subu
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: jal __sync_fetch_and_sub_8
; MIPS32: sync
define internal i32 @test_atomic_rmw_sub_32_ignored(i32 %iptr, i32 %v) {
......@@ -1058,14 +1023,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_rmw_or_64
; MIPS32: sync
; MIPS32: ll
; MIPS32: or
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: ll
; MIPS32: or
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: jal __sync_fetch_and_or_8
; MIPS32: sync
define internal i32 @test_atomic_rmw_or_32_ignored(i32 %iptr, i32 %v) {
......@@ -1237,14 +1195,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_rmw_and_64
; MIPS32: sync
; MIPS32: ll
; MIPS32: and
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: ll
; MIPS32: and
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: jal __sync_fetch_and_and_8
; MIPS32: sync
define internal i32 @test_atomic_rmw_and_32_ignored(i32 %iptr, i32 %v) {
......@@ -1414,14 +1365,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_rmw_xor_64
; MIPS32: sync
; MIPS32: ll
; MIPS32: xor
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: ll
; MIPS32: xor
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: jal __sync_fetch_and_xor_8
; MIPS32: sync
define internal i32 @test_atomic_rmw_xor_32_ignored(i32 %iptr, i32 %v) {
......@@ -1574,14 +1518,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_rmw_xchg_64
; MIPS32: sync
; MIPS32: ll
; MIPS32: move
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: ll
; MIPS32: move
; MIPS32: sc
; MIPS32: beq {{.*}}, $zero, {{.*}}
; MIPS32: jal __sync_lock_test_and_set_8
; MIPS32: sync
define internal i32 @test_atomic_rmw_xchg_32_ignored(i32 %iptr, i32 %v) {
......@@ -1769,14 +1706,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_cmpxchg_64
; MIPS32: sync
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: jal __sync_val_compare_and_swap_8
; MIPS32: sync
......@@ -1803,14 +1733,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_cmpxchg_64_undef
; MIPS32: sync
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: jal __sync_val_compare_and_swap_8
; MIPS32: sync
; Test a case where %old really does need to be copied out of edx:eax.
......@@ -1847,14 +1770,7 @@ entry:
; ARM32: str
; MIPS32-LABEL: test_atomic_cmpxchg_64_store
; MIPS32: sync
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: jal __sync_val_compare_and_swap_8
; MIPS32: sync
......@@ -1902,14 +1818,7 @@ eblock:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_cmpxchg_64_alloca
; MIPS32: sync
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: jal __sync_val_compare_and_swap_8
; MIPS32: sync
define internal i32 @test_atomic_cmpxchg_32_ignored(i32 %iptr, i32 %expected,
......@@ -1967,14 +1876,7 @@ entry:
; ARM32: dmb
; MIPS32-LABEL: test_atomic_cmpxchg_64_ignored
; MIPS32: sync
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: ll
; MIPS32: bne
; MIPS32: sc
; MIPS32: beq
; MIPS32: jal __sync_val_compare_and_swap_8
; MIPS32: sync
;;;; Fence and is-lock-free.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment