Skip to content

S
swiftshader
Commit cac003e8 by Jim Stichnoth
Options

Subzero: Add more kinds of RMW lowering.

Specifically: sub, and, or, xor; for all integer types. Turns out that RMW is not possible for fadd/fsub/fmul/fdiv as well as operations on vector types, because the corresponding x86 instructions require the result to be in a physical register. Refactors the assembler's implementations of add/or/adc/sbb/and/sub/xor/cmp to avoid repetition. BUG= https://code.google.com/p/nativeclient/issues/detail?id=4095 R=jvoung@chromium.org Review URL: https://codereview.chromium.org/1186713010
parent a9eeb420
Showing with 151 additions and 25 deletions
src/IceAssemblerX8632.h
......@@ -708,14 +708,20 @@ public:
void And(Type Ty, GPRRegister dst, GPRRegister src);
void And(Type Ty, GPRRegister dst, const Address &address);
void And(Type Ty, GPRRegister dst, const Immediate &imm);
void And(Type Ty, const Address &address, GPRRegister reg);
void And(Type Ty, const Address &address, const Immediate &imm);
void Or(Type Ty, GPRRegister dst, GPRRegister src);
void Or(Type Ty, GPRRegister dst, const Address &address);
void Or(Type Ty, GPRRegister dst, const Immediate &imm);
void Or(Type Ty, const Address &address, GPRRegister reg);
void Or(Type Ty, const Address &address, const Immediate &imm);
void Xor(Type Ty, GPRRegister dst, GPRRegister src);
void Xor(Type Ty, GPRRegister dst, const Address &address);
void Xor(Type Ty, GPRRegister dst, const Immediate &imm);
void Xor(Type Ty, const Address &address, GPRRegister reg);
void Xor(Type Ty, const Address &address, const Immediate &imm);
void add(Type Ty, GPRRegister dst, GPRRegister src);
void add(Type Ty, GPRRegister reg, const Address &address);
......@@ -726,14 +732,20 @@ public:
void adc(Type Ty, GPRRegister dst, GPRRegister src);
void adc(Type Ty, GPRRegister dst, const Address &address);
void adc(Type Ty, GPRRegister reg, const Immediate &imm);
void adc(Type Ty, const Address &address, GPRRegister reg);
void adc(Type Ty, const Address &address, const Immediate &imm);
void sub(Type Ty, GPRRegister dst, GPRRegister src);
void sub(Type Ty, GPRRegister reg, const Address &address);
void sub(Type Ty, GPRRegister reg, const Immediate &imm);
void sub(Type Ty, const Address &address, GPRRegister reg);
void sub(Type Ty, const Address &address, const Immediate &imm);
void sbb(Type Ty, GPRRegister dst, GPRRegister src);
void sbb(Type Ty, GPRRegister reg, const Address &address);
void sbb(Type Ty, GPRRegister reg, const Immediate &imm);
void sbb(Type Ty, const Address &address, GPRRegister reg);
void sbb(Type Ty, const Address &address, const Immediate &imm);
void cbw();
void cwd();
......@@ -859,6 +871,24 @@ private:
LabelVector LocalLabels;
Label *GetOrCreateLabel(SizeT Number, LabelVector &Labels);
// The arith_int() methods factor out the commonality between the encodings of
// add(), Or(), adc(), sbb(), And(), sub(), Xor(), and cmp(). The Tag
// parameter is statically asserted to be less than 8.
template <uint32_t Tag>
void arith_int(Type Ty, GPRRegister reg, const Immediate &imm);
template <uint32_t Tag>
void arith_int(Type Ty, GPRRegister reg0, GPRRegister reg1);
template <uint32_t Tag>
void arith_int(Type Ty, GPRRegister reg, const Address &address);
template <uint32_t Tag>
void arith_int(Type Ty, const Address &address, GPRRegister reg);
template <uint32_t Tag>
void arith_int(Type Ty, const Address &address, const Immediate &imm);
};
inline void AssemblerX8632::emitUint8(uint8_t value) {
......
src/IceELFObjectWriter.cpp
......@@ -390,8 +390,9 @@ void ELFObjectWriter::writeDataOfType(SectionType ST,
for (VariableDeclaration::Initializer *Init : Var->getInitializers()) {
switch (Init->getKind()) {
case VariableDeclaration::Initializer::DataInitializerKind: {
const auto Data = llvm::cast<VariableDeclaration::DataInitializer>(
Init)->getContents();
const auto Data =
llvm::cast<VariableDeclaration::DataInitializer>(Init)
->getContents();
Section->appendData(Str, llvm::StringRef(Data.data(), Data.size()));
break;
}
......
src/IceGlobalContext.cpp
......@@ -364,9 +364,7 @@ void GlobalContext::emitFileHeader() {
}
}
void GlobalContext::lowerConstants() {
DataLowering->lowerConstants();
}
void GlobalContext::lowerConstants() { DataLowering->lowerConstants(); }
void GlobalContext::lowerGlobals(const IceString &SectionSuffix) {
TimerMarker T(TimerStack::TT_emitGlobalInitializers, this);
......
src/IceInstX8632.cpp
......@@ -937,19 +937,25 @@ template <> const char *InstX8632Add::Opcode = "add";
template <> const char *InstX8632AddRMW::Opcode = "add";
template <> const char *InstX8632Addps::Opcode = "addps";
template <> const char *InstX8632Adc::Opcode = "adc";
template <> const char *InstX8632AdcRMW::Opcode = "adc";
template <> const char *InstX8632Addss::Opcode = "addss";
template <> const char *InstX8632Padd::Opcode = "padd";
template <> const char *InstX8632Sub::Opcode = "sub";
template <> const char *InstX8632SubRMW::Opcode = "sub";
template <> const char *InstX8632Subps::Opcode = "subps";
template <> const char *InstX8632Subss::Opcode = "subss";
template <> const char *InstX8632Sbb::Opcode = "sbb";
template <> const char *InstX8632SbbRMW::Opcode = "sbb";
template <> const char *InstX8632Psub::Opcode = "psub";
template <> const char *InstX8632And::Opcode = "and";
template <> const char *InstX8632AndRMW::Opcode = "and";
template <> const char *InstX8632Pand::Opcode = "pand";
template <> const char *InstX8632Pandn::Opcode = "pandn";
template <> const char *InstX8632Or::Opcode = "or";
template <> const char *InstX8632OrRMW::Opcode = "or";
template <> const char *InstX8632Por::Opcode = "por";
template <> const char *InstX8632Xor::Opcode = "xor";
template <> const char *InstX8632XorRMW::Opcode = "xor";
template <> const char *InstX8632Pxor::Opcode = "pxor";
template <> const char *InstX8632Imul::Opcode = "imul";
template <> const char *InstX8632Mulps::Opcode = "mulps";
......@@ -1025,25 +1031,43 @@ const X8632::AssemblerX8632::GPREmitterRegOp InstX8632Adc::Emitter = {
&X8632::AssemblerX8632::adc, &X8632::AssemblerX8632::adc,
&X8632::AssemblerX8632::adc};
template <>
const X8632::AssemblerX8632::GPREmitterAddrOp InstX8632AdcRMW::Emitter = {
&X8632::AssemblerX8632::adc, &X8632::AssemblerX8632::adc};
template <>
const X8632::AssemblerX8632::GPREmitterRegOp InstX8632And::Emitter = {
&X8632::AssemblerX8632::And, &X8632::AssemblerX8632::And,
&X8632::AssemblerX8632::And};
template <>
const X8632::AssemblerX8632::GPREmitterAddrOp InstX8632AndRMW::Emitter = {
&X8632::AssemblerX8632::And, &X8632::AssemblerX8632::And};
template <>
const X8632::AssemblerX8632::GPREmitterRegOp InstX8632Or::Emitter = {
&X8632::AssemblerX8632::Or, &X8632::AssemblerX8632::Or,
&X8632::AssemblerX8632::Or};
template <>
const X8632::AssemblerX8632::GPREmitterAddrOp InstX8632OrRMW::Emitter = {
&X8632::AssemblerX8632::Or, &X8632::AssemblerX8632::Or};
template <>
const X8632::AssemblerX8632::GPREmitterRegOp InstX8632Sbb::Emitter = {
&X8632::AssemblerX8632::sbb, &X8632::AssemblerX8632::sbb,
&X8632::AssemblerX8632::sbb};
template <>
const X8632::AssemblerX8632::GPREmitterAddrOp InstX8632SbbRMW::Emitter = {
&X8632::AssemblerX8632::sbb, &X8632::AssemblerX8632::sbb};
template <>
const X8632::AssemblerX8632::GPREmitterRegOp InstX8632Sub::Emitter = {
&X8632::AssemblerX8632::sub, &X8632::AssemblerX8632::sub,
&X8632::AssemblerX8632::sub};
template <>
const X8632::AssemblerX8632::GPREmitterAddrOp InstX8632SubRMW::Emitter = {
&X8632::AssemblerX8632::sub, &X8632::AssemblerX8632::sub};
template <>
const X8632::AssemblerX8632::GPREmitterRegOp InstX8632Xor::Emitter = {
&X8632::AssemblerX8632::Xor, &X8632::AssemblerX8632::Xor,
&X8632::AssemblerX8632::Xor};
template <>
const X8632::AssemblerX8632::GPREmitterAddrOp InstX8632XorRMW::Emitter = {
&X8632::AssemblerX8632::Xor, &X8632::AssemblerX8632::Xor};
// Binary Shift GPR ops
template <>
......
src/IceInstX8632.h
......@@ -175,12 +175,14 @@ public:
enum InstKindX8632 {
k__Start = Inst::Target,
Adc,
AdcRMW,
Add,
AddRMW,
Addps,
Addss,
Adjuststack,
And,
AndRMW,
Blendvps,
Br,
Bsf,
......@@ -221,6 +223,7 @@ public:
Neg,
Nop,
Or,
OrRMW,
Padd,
Pand,
Pandn,
......@@ -244,6 +247,7 @@ public:
Rol,
Sar,
Sbb,
SbbRMW,
Setcc,
Shl,
Shld,
......@@ -255,6 +259,7 @@ public:
StoreP,
StoreQ,
Sub,
SubRMW,
Subps,
Subss,
Test,
......@@ -262,7 +267,8 @@ public:
UD2,
Xadd,
Xchg,
Xor
Xor,
XorRMW
};
static const char *getWidthString(Type Ty);
......@@ -1103,19 +1109,25 @@ typedef InstX8632BinopGPR<InstX8632::Add> InstX8632Add;
typedef InstX8632BinopRMW<InstX8632::AddRMW> InstX8632AddRMW;
typedef InstX8632BinopXmm<InstX8632::Addps, true> InstX8632Addps;
typedef InstX8632BinopGPR<InstX8632::Adc> InstX8632Adc;
typedef InstX8632BinopRMW<InstX8632::AdcRMW> InstX8632AdcRMW;
typedef InstX8632BinopXmm<InstX8632::Addss, false> InstX8632Addss;
typedef InstX8632BinopXmm<InstX8632::Padd, true> InstX8632Padd;
typedef InstX8632BinopGPR<InstX8632::Sub> InstX8632Sub;
typedef InstX8632BinopRMW<InstX8632::SubRMW> InstX8632SubRMW;
typedef InstX8632BinopXmm<InstX8632::Subps, true> InstX8632Subps;
typedef InstX8632BinopXmm<InstX8632::Subss, false> InstX8632Subss;
typedef InstX8632BinopGPR<InstX8632::Sbb> InstX8632Sbb;
typedef InstX8632BinopRMW<InstX8632::SbbRMW> InstX8632SbbRMW;
typedef InstX8632BinopXmm<InstX8632::Psub, true> InstX8632Psub;
typedef InstX8632BinopGPR<InstX8632::And> InstX8632And;
typedef InstX8632BinopRMW<InstX8632::AndRMW> InstX8632AndRMW;
typedef InstX8632BinopXmm<InstX8632::Pand, false> InstX8632Pand;
typedef InstX8632BinopXmm<InstX8632::Pandn, false> InstX8632Pandn;
typedef InstX8632BinopGPR<InstX8632::Or> InstX8632Or;
typedef InstX8632BinopRMW<InstX8632::OrRMW> InstX8632OrRMW;
typedef InstX8632BinopXmm<InstX8632::Por, false> InstX8632Por;
typedef InstX8632BinopGPR<InstX8632::Xor> InstX8632Xor;
typedef InstX8632BinopRMW<InstX8632::XorRMW> InstX8632XorRMW;
typedef InstX8632BinopXmm<InstX8632::Pxor, false> InstX8632Pxor;
typedef InstX8632BinopGPR<InstX8632::Imul> InstX8632Imul;
typedef InstX8632BinopXmm<InstX8632::Mulps, true> InstX8632Mulps;
......
src/IceTargetLowering.cpp
......@@ -517,8 +517,9 @@ void TargetDataLowering::emitGlobal(const VariableDeclaration &Var,
for (VariableDeclaration::Initializer *Init : Var.getInitializers()) {
switch (Init->getKind()) {
case VariableDeclaration::Initializer::DataInitializerKind: {
const auto &Data = llvm::cast<VariableDeclaration::DataInitializer>(
Init)->getContents();
const auto &Data =
llvm::cast<VariableDeclaration::DataInitializer>(Init)
->getContents();
for (SizeT i = 0; i < Init->getNumBytes(); ++i) {
Str << "\t.byte\t" << (((unsigned)Data[i]) & 0xff) << "\n";
}
......
src/IceTargetLoweringX8632.cpp
......@@ -587,32 +587,31 @@ namespace {
bool canRMW(const InstArithmetic *Arith) {
Type Ty = Arith->getDest()->getType();
// X86 vector instructions write to a register and have no RMW
// option.
if (isVectorType(Ty))
return false;
bool isI64 = Ty == IceType_i64;
bool isVector = isVectorType(Ty);
switch (Arith->getOp()) {
// Not handled for lack of simple lowering:
// shift on i64 and vectors
// shift on i64
// mul, udiv, urem, sdiv, srem, frem
// Not handled for lack of RMW instructions:
// fadd, fsub, fmul, fdiv (also vector types)
default:
return false;
case InstArithmetic::Add:
return !isI64 && !isVector; // TODO(stichnot): implement i64 and vector
case InstArithmetic::Sub:
case InstArithmetic::And:
case InstArithmetic::Or:
case InstArithmetic::Xor:
case InstArithmetic::Fadd:
case InstArithmetic::Fsub:
case InstArithmetic::Fmul:
case InstArithmetic::Fdiv:
return false; // TODO(stichnot): implement
return true;
case InstArithmetic::Shl:
case InstArithmetic::Lshr:
case InstArithmetic::Ashr:
return false; // TODO(stichnot): implement
return !isI64 && !isVector;
return !isI64;
}
}
......@@ -3442,10 +3441,11 @@ void TargetX8632::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
Func->setError("Unexpected memory ordering for AtomicRMW");
return;
}
lowerAtomicRMW(Instr->getDest(),
static_cast<uint32_t>(llvm::cast<ConstantInteger32>(
Instr->getArg(0))->getValue()),
Instr->getArg(1), Instr->getArg(2));
lowerAtomicRMW(
Instr->getDest(),
static_cast<uint32_t>(
llvm::cast<ConstantInteger32>(Instr->getArg(0))->getValue()),
Instr->getArg(1), Instr->getArg(2));
return;
case Intrinsics::AtomicStore: {
if (!Intrinsics::isMemoryOrderValid(
......@@ -4660,11 +4660,37 @@ void TargetX8632::lowerRMW(const InstX8632FakeRMW *RMW) {
Type Ty = Src->getType();
OperandX8632Mem *Addr = formMemoryOperand(RMW->getAddr(), Ty);
if (Ty == IceType_i64) {
// TODO(stichnot): Implement.
} else if (isVectorType(Ty)) {
// TODO(stichnot): Implement.
Operand *SrcLo = legalize(loOperand(Src), Legal_Reg | Legal_Imm);
Operand *SrcHi = legalize(hiOperand(Src), Legal_Reg | Legal_Imm);
OperandX8632Mem *AddrLo = llvm::cast<OperandX8632Mem>(loOperand(Addr));
OperandX8632Mem *AddrHi = llvm::cast<OperandX8632Mem>(hiOperand(Addr));
switch (RMW->getOp()) {
default:
// TODO(stichnot): Implement other arithmetic operators.
break;
case InstArithmetic::Add:
_add_rmw(AddrLo, SrcLo);
_adc_rmw(AddrHi, SrcHi);
return;
case InstArithmetic::Sub:
_sub_rmw(AddrLo, SrcLo);
_sbb_rmw(AddrHi, SrcHi);
return;
case InstArithmetic::And:
_and_rmw(AddrLo, SrcLo);
_and_rmw(AddrHi, SrcHi);
return;
case InstArithmetic::Or:
_or_rmw(AddrLo, SrcLo);
_or_rmw(AddrHi, SrcHi);
return;
case InstArithmetic::Xor:
_xor_rmw(AddrLo, SrcLo);
_xor_rmw(AddrHi, SrcHi);
return;
}
} else {
// i8, i16, i32, f32, f64
// i8, i16, i32
switch (RMW->getOp()) {
default:
// TODO(stichnot): Implement other arithmetic operators.
......@@ -4673,6 +4699,22 @@ void TargetX8632::lowerRMW(const InstX8632FakeRMW *RMW) {
Src = legalize(Src, Legal_Reg | Legal_Imm);
_add_rmw(Addr, Src);
return;
case InstArithmetic::Sub:
Src = legalize(Src, Legal_Reg | Legal_Imm);
_sub_rmw(Addr, Src);
return;
case InstArithmetic::And:
Src = legalize(Src, Legal_Reg | Legal_Imm);
_and_rmw(Addr, Src);
return;
case InstArithmetic::Or:
Src = legalize(Src, Legal_Reg | Legal_Imm);
_or_rmw(Addr, Src);
return;
case InstArithmetic::Xor:
Src = legalize(Src, Legal_Reg | Legal_Imm);
_xor_rmw(Addr, Src);
return;
}
}
llvm::report_fatal_error("Couldn't lower RMW instruction");
......
src/IceTargetLoweringX8632.h
......@@ -264,6 +264,9 @@ protected:
void _adc(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Adc::create(Func, Dest, Src0));
}
void _adc_rmw(OperandX8632Mem *DestSrc0, Operand *Src1) {
Context.insert(InstX8632AdcRMW::create(Func, DestSrc0, Src1));
}
void _add(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Add::create(Func, Dest, Src0));
}
......@@ -283,6 +286,9 @@ protected:
void _and(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632And::create(Func, Dest, Src0));
}
void _and_rmw(OperandX8632Mem *DestSrc0, Operand *Src1) {
Context.insert(InstX8632AndRMW::create(Func, DestSrc0, Src1));
}
void _blendvps(Variable *Dest, Operand *Src0, Operand *Src1) {
Context.insert(InstX8632Blendvps::create(Func, Dest, Src0, Src1));
}
......@@ -425,6 +431,9 @@ protected:
void _or(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Or::create(Func, Dest, Src0));
}
void _or_rmw(OperandX8632Mem *DestSrc0, Operand *Src1) {
Context.insert(InstX8632OrRMW::create(Func, DestSrc0, Src1));
}
void _padd(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Padd::create(Func, Dest, Src0));
}
......@@ -494,6 +503,9 @@ protected:
void _sbb(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Sbb::create(Func, Dest, Src0));
}
void _sbb_rmw(OperandX8632Mem *DestSrc0, Operand *Src1) {
Context.insert(InstX8632SbbRMW::create(Func, DestSrc0, Src1));
}
void _setcc(Variable *Dest, CondX86::BrCond Condition) {
Context.insert(InstX8632Setcc::create(Func, Dest, Condition));
}
......@@ -527,6 +539,9 @@ protected:
void _sub(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Sub::create(Func, Dest, Src0));
}
void _sub_rmw(OperandX8632Mem *DestSrc0, Operand *Src1) {
Context.insert(InstX8632SubRMW::create(Func, DestSrc0, Src1));
}
void _subps(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Subps::create(Func, Dest, Src0));
}
......@@ -561,6 +576,9 @@ protected:
void _xor(Variable *Dest, Operand *Src0) {
Context.insert(InstX8632Xor::create(Func, Dest, Src0));
}
void _xor_rmw(OperandX8632Mem *DestSrc0, Operand *Src1) {
Context.insert(InstX8632XorRMW::create(Func, DestSrc0, Src1));
}
void _set_dest_nonkillable() {
Context.getLastInserted()->setDestNonKillable();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment