Skip to content

S
swiftshader
Commit f1f773dd by Jim Stichnoth
Options

Subzero: Fix over-aggressive bool folding.

The problem is that bitcode like this: %cond = cmp %var, [mem] store ..., mem br cond, target1, target2 would be bool-folded into this: //deleted cmp store ..., mem br (%var==[mem]), target1, target2 And if the memory operands point to the same location, results are incorrect. In addition to stores, this is a problem for RMW instructions, and most call instructions which could perform stores before returning. BUG= https://bugs.chromium.org/p/nativeclient/issues/detail?id=4370 R=eholk@chromium.org, jpp@chromium.org Review URL: https://codereview.chromium.org/1904233002 .
parent a47c11c7
Showing with 180 additions and 33 deletions
src/IceInst.cpp
...@@ -189,6 +189,10 @@ void Inst::spliceLivenessInfo(Inst *OrigInst, Inst *SpliceAssn) { ...@@ -189,6 +189,10 @@ void Inst::spliceLivenessInfo(Inst *OrigInst, Inst *SpliceAssn) {
llvm::report_fatal_error("Failed to find splice operand"); llvm::report_fatal_error("Failed to find splice operand");
} }
bool Inst::isMemoryWrite() const {
llvm::report_fatal_error("Attempt to call base Inst::isMemoryWrite() method");
}
void Inst::livenessLightweight(Cfg *Func, LivenessBV &Live) { void Inst::livenessLightweight(Cfg *Func, LivenessBV &Live) {
assert(!isDeleted()); assert(!isDeleted());
resetLastUses(); resetLastUses();
......
src/IceInst.h
...@@ -137,6 +137,13 @@ public: ...@@ -137,6 +137,13 @@ public:
/// "var_dest=var_src" assignment where the dest and src are both variables. /// "var_dest=var_src" assignment where the dest and src are both variables.
virtual bool isVarAssign() const { return false; } virtual bool isVarAssign() const { return false; }
/// Returns true if the instruction has a possible side effect of changing
/// memory, in which case a memory load should not be reordered with respect
/// to this instruction. It should really be pure virtual, but we can't
/// because of g++ and llvm::ilist<>, so we implement it as
/// report_fatal_error().
virtual bool isMemoryWrite() const;
void livenessLightweight(Cfg *Func, LivenessBV &Live); void livenessLightweight(Cfg *Func, LivenessBV &Live);
/// Calculates liveness for this instruction. Returns true if this instruction /// Calculates liveness for this instruction. Returns true if this instruction
/// is (tentatively) still live and should be retained, and false if this /// is (tentatively) still live and should be retained, and false if this
...@@ -261,6 +268,7 @@ public: ...@@ -261,6 +268,7 @@ public:
Operand *getSizeInBytes() const { return getSrc(0); } Operand *getSizeInBytes() const { return getSrc(0); }
bool getKnownFrameOffset() const { return KnownFrameOffset; } bool getKnownFrameOffset() const { return KnownFrameOffset; }
void setKnownFrameOffset() { KnownFrameOffset = true; } void setKnownFrameOffset() { KnownFrameOffset = true; }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Alloca; } static bool classof(const Inst *Instr) { return Instr->getKind() == Alloca; }
...@@ -298,6 +306,7 @@ public: ...@@ -298,6 +306,7 @@ public:
static const char *getOpName(OpKind Op); static const char *getOpName(OpKind Op);
bool isCommutative() const; bool isCommutative() const;
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { static bool classof(const Inst *Instr) {
return Instr->getKind() == Arithmetic; return Instr->getKind() == Arithmetic;
...@@ -325,6 +334,7 @@ public: ...@@ -325,6 +334,7 @@ public:
return new (Func->allocate<InstAssign>()) InstAssign(Func, Dest, Source); return new (Func->allocate<InstAssign>()) InstAssign(Func, Dest, Source);
} }
bool isVarAssign() const override; bool isVarAssign() const override;
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Assign; } static bool classof(const Inst *Instr) { return Instr->getKind() == Assign; }
...@@ -365,6 +375,7 @@ public: ...@@ -365,6 +375,7 @@ public:
NodeList getTerminatorEdges() const override; NodeList getTerminatorEdges() const override;
bool isUnconditionalBranch() const override { return isUnconditional(); } bool isUnconditionalBranch() const override { return isUnconditional(); }
bool repointEdges(CfgNode *OldNode, CfgNode *NewNode) override; bool repointEdges(CfgNode *OldNode, CfgNode *NewNode) override;
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Br; } static bool classof(const Inst *Instr) { return Instr->getKind() == Br; }
...@@ -404,6 +415,7 @@ public: ...@@ -404,6 +415,7 @@ public:
SizeT getNumArgs() const { return getSrcSize() - 1; } SizeT getNumArgs() const { return getSrcSize() - 1; }
bool isTailcall() const { return HasTailCall; } bool isTailcall() const { return HasTailCall; }
bool isTargetHelperCall() const { return IsTargetHelperCall; } bool isTargetHelperCall() const { return IsTargetHelperCall; }
bool isMemoryWrite() const override { return true; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Call; } static bool classof(const Inst *Instr) { return Instr->getKind() == Call; }
Type getReturnType() const; Type getReturnType() const;
...@@ -445,6 +457,7 @@ public: ...@@ -445,6 +457,7 @@ public:
InstCast(Func, CastKind, Dest, Source); InstCast(Func, CastKind, Dest, Source);
} }
OpKind getCastKind() const { return CastKind; } OpKind getCastKind() const { return CastKind; }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Cast; } static bool classof(const Inst *Instr) { return Instr->getKind() == Cast; }
...@@ -467,6 +480,7 @@ public: ...@@ -467,6 +480,7 @@ public:
InstExtractElement(Func, Dest, Source1, Source2); InstExtractElement(Func, Dest, Source1, Source2);
} }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { static bool classof(const Inst *Instr) {
return Instr->getKind() == ExtractElement; return Instr->getKind() == ExtractElement;
...@@ -498,6 +512,7 @@ public: ...@@ -498,6 +512,7 @@ public:
InstFcmp(Func, Condition, Dest, Source1, Source2); InstFcmp(Func, Condition, Dest, Source1, Source2);
} }
FCond getCondition() const { return Condition; } FCond getCondition() const { return Condition; }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Fcmp; } static bool classof(const Inst *Instr) { return Instr->getKind() == Fcmp; }
...@@ -529,6 +544,7 @@ public: ...@@ -529,6 +544,7 @@ public:
InstIcmp(Func, Condition, Dest, Source1, Source2); InstIcmp(Func, Condition, Dest, Source1, Source2);
} }
ICond getCondition() const { return Condition; } ICond getCondition() const { return Condition; }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Icmp; } static bool classof(const Inst *Instr) { return Instr->getKind() == Icmp; }
...@@ -552,6 +568,7 @@ public: ...@@ -552,6 +568,7 @@ public:
InstInsertElement(Func, Dest, Source1, Source2, Source3); InstInsertElement(Func, Dest, Source1, Source2, Source3);
} }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { static bool classof(const Inst *Instr) {
return Instr->getKind() == InsertElement; return Instr->getKind() == InsertElement;
...@@ -581,6 +598,9 @@ public: ...@@ -581,6 +598,9 @@ public:
} }
Intrinsics::IntrinsicInfo getIntrinsicInfo() const { return Info; } Intrinsics::IntrinsicInfo getIntrinsicInfo() const { return Info; }
bool isMemoryWrite() const override {
return getIntrinsicInfo().IsMemoryWrite;
}
private: private:
InstIntrinsicCall(Cfg *Func, SizeT NumArgs, Variable *Dest, InstIntrinsicCall(Cfg *Func, SizeT NumArgs, Variable *Dest,
...@@ -606,6 +626,7 @@ public: ...@@ -606,6 +626,7 @@ public:
return new (Func->allocate<InstLoad>()) InstLoad(Func, Dest, SourceAddr); return new (Func->allocate<InstLoad>()) InstLoad(Func, Dest, SourceAddr);
} }
Operand *getSourceAddress() const { return getSrc(0); } Operand *getSourceAddress() const { return getSrc(0); }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Load; } static bool classof(const Inst *Instr) { return Instr->getKind() == Load; }
...@@ -632,6 +653,7 @@ public: ...@@ -632,6 +653,7 @@ public:
void livenessPhiOperand(LivenessBV &Live, CfgNode *Target, void livenessPhiOperand(LivenessBV &Live, CfgNode *Target,
Liveness *Liveness); Liveness *Liveness);
Inst *lower(Cfg *Func); Inst *lower(Cfg *Func);
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Phi; } static bool classof(const Inst *Instr) { return Instr->getKind() == Phi; }
...@@ -666,6 +688,7 @@ public: ...@@ -666,6 +688,7 @@ public:
return getSrc(0); return getSrc(0);
} }
NodeList getTerminatorEdges() const override { return NodeList(); } NodeList getTerminatorEdges() const override { return NodeList(); }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Ret; } static bool classof(const Inst *Instr) { return Instr->getKind() == Ret; }
...@@ -688,6 +711,7 @@ public: ...@@ -688,6 +711,7 @@ public:
Operand *getCondition() const { return getSrc(0); } Operand *getCondition() const { return getSrc(0); }
Operand *getTrueOperand() const { return getSrc(1); } Operand *getTrueOperand() const { return getSrc(1); }
Operand *getFalseOperand() const { return getSrc(2); } Operand *getFalseOperand() const { return getSrc(2); }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Select; } static bool classof(const Inst *Instr) { return Instr->getKind() == Select; }
...@@ -714,6 +738,7 @@ public: ...@@ -714,6 +738,7 @@ public:
Operand *getData() const { return getSrc(0); } Operand *getData() const { return getSrc(0); }
Variable *getRmwBeacon() const; Variable *getRmwBeacon() const;
void setRmwBeacon(Variable *Beacon); void setRmwBeacon(Variable *Beacon);
bool isMemoryWrite() const override { return true; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Store; } static bool classof(const Inst *Instr) { return Instr->getKind() == Store; }
...@@ -747,6 +772,7 @@ public: ...@@ -747,6 +772,7 @@ public:
void addBranch(SizeT CaseIndex, uint64_t Value, CfgNode *Label); void addBranch(SizeT CaseIndex, uint64_t Value, CfgNode *Label);
NodeList getTerminatorEdges() const override; NodeList getTerminatorEdges() const override;
bool repointEdges(CfgNode *OldNode, CfgNode *NewNode) override; bool repointEdges(CfgNode *OldNode, CfgNode *NewNode) override;
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == Switch; } static bool classof(const Inst *Instr) { return Instr->getKind() == Switch; }
...@@ -775,6 +801,7 @@ public: ...@@ -775,6 +801,7 @@ public:
return new (Func->allocate<InstUnreachable>()) InstUnreachable(Func); return new (Func->allocate<InstUnreachable>()) InstUnreachable(Func);
} }
NodeList getTerminatorEdges() const override { return NodeList(); } NodeList getTerminatorEdges() const override { return NodeList(); }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { static bool classof(const Inst *Instr) {
return Instr->getKind() == Unreachable; return Instr->getKind() == Unreachable;
...@@ -799,6 +826,7 @@ public: ...@@ -799,6 +826,7 @@ public:
} }
void emit(const Cfg *Func) const override; void emit(const Cfg *Func) const override;
void emitIAS(const Cfg * /* Func */) const override {} void emitIAS(const Cfg * /* Func */) const override {}
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
Option getOption() const { return BundleOption; } Option getOption() const { return BundleOption; }
static bool classof(const Inst *Instr) { static bool classof(const Inst *Instr) {
...@@ -822,6 +850,7 @@ public: ...@@ -822,6 +850,7 @@ public:
} }
void emit(const Cfg *Func) const override; void emit(const Cfg *Func) const override;
void emitIAS(const Cfg * /* Func */) const override {} void emitIAS(const Cfg * /* Func */) const override {}
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { static bool classof(const Inst *Instr) {
return Instr->getKind() == BundleUnlock; return Instr->getKind() == BundleUnlock;
...@@ -854,6 +883,7 @@ public: ...@@ -854,6 +883,7 @@ public:
} }
void emit(const Cfg *Func) const override; void emit(const Cfg *Func) const override;
void emitIAS(const Cfg * /* Func */) const override {} void emitIAS(const Cfg * /* Func */) const override {}
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == FakeDef; } static bool classof(const Inst *Instr) { return Instr->getKind() == FakeDef; }
...@@ -879,6 +909,7 @@ public: ...@@ -879,6 +909,7 @@ public:
} }
void emit(const Cfg *Func) const override; void emit(const Cfg *Func) const override;
void emitIAS(const Cfg * /* Func */) const override {} void emitIAS(const Cfg * /* Func */) const override {}
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() == FakeUse; } static bool classof(const Inst *Instr) { return Instr->getKind() == FakeUse; }
...@@ -907,6 +938,7 @@ public: ...@@ -907,6 +938,7 @@ public:
const Inst *getLinked() const { return Linked; } const Inst *getLinked() const { return Linked; }
void emit(const Cfg *Func) const override; void emit(const Cfg *Func) const override;
void emitIAS(const Cfg * /* Func */) const override {} void emitIAS(const Cfg * /* Func */) const override {}
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { static bool classof(const Inst *Instr) {
return Instr->getKind() == FakeKill; return Instr->getKind() == FakeKill;
...@@ -989,6 +1021,7 @@ public: ...@@ -989,6 +1021,7 @@ public:
assert(I < NumTargets); assert(I < NumTargets);
return Targets[I]; return Targets[I];
} }
bool isMemoryWrite() const override { return false; }
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { static bool classof(const Inst *Instr) {
return Instr->getKind() == JumpTable; return Instr->getKind() == JumpTable;
...@@ -1036,7 +1069,8 @@ public: ...@@ -1036,7 +1069,8 @@ public:
InstBreakpoint(const InstBreakpoint &) = delete; InstBreakpoint(const InstBreakpoint &) = delete;
InstBreakpoint &operator=(const InstBreakpoint &) = delete; InstBreakpoint &operator=(const InstBreakpoint &) = delete;
InstBreakpoint(Cfg *Func); explicit InstBreakpoint(Cfg *Func);
bool isMemoryWrite() const override { return false; }
public: public:
static InstBreakpoint *create(Cfg *Func) { static InstBreakpoint *create(Cfg *Func) {
...@@ -1057,6 +1091,9 @@ class InstTarget : public Inst { ...@@ -1057,6 +1091,9 @@ class InstTarget : public Inst {
public: public:
uint32_t getEmitInstCount() const override { return 1; } uint32_t getEmitInstCount() const override { return 1; }
bool isMemoryWrite() const override {
return true; // conservative answer
}
void dump(const Cfg *Func) const override; void dump(const Cfg *Func) const override;
static bool classof(const Inst *Instr) { return Instr->getKind() >= Target; } static bool classof(const Inst *Instr) { return Instr->getKind() >= Target; }
......
src/IceIntrinsics.cpp
...@@ -31,8 +31,8 @@ static_assert(sizeof(Intrinsics::IntrinsicInfo) == 4, ...@@ -31,8 +31,8 @@ static_assert(sizeof(Intrinsics::IntrinsicInfo) == 4,
namespace { namespace {
#define INTRIN(ID, SE, RT) \ #define INTRIN(ID, SE, RT, MW) \
{ Intrinsics::ID, Intrinsics::SE, Intrinsics::RT } { Intrinsics::ID, Intrinsics::SE, Intrinsics::RT, Intrinsics::MW }
// Build list of intrinsics with their attributes and expected prototypes. List // Build list of intrinsics with their attributes and expected prototypes. List
// is sorted alphabetically. // is sorted alphabetically.
...@@ -44,12 +44,8 @@ const struct IceIntrinsicsEntry_ { ...@@ -44,12 +44,8 @@ const struct IceIntrinsicsEntry_ {
#define AtomicCmpxchgInit(Overload, NameSuffix) \ #define AtomicCmpxchgInit(Overload, NameSuffix) \
{ \ { \
{ \ { \
INTRIN(AtomicCmpxchg, SideEffects_T, ReturnsTwice_F), {Overload, \ INTRIN(AtomicCmpxchg, SideEffects_T, ReturnsTwice_F, MemoryWrite_T), \
IceType_i32, \ {Overload, IceType_i32, Overload, Overload, IceType_i32, IceType_i32}, \
Overload, \
Overload, \
IceType_i32, \
IceType_i32}, \
6 \ 6 \
} \ } \
, "llvm.nacl.atomic.cmpxchg." NameSuffix \ , "llvm.nacl.atomic.cmpxchg." NameSuffix \
...@@ -60,13 +56,15 @@ const struct IceIntrinsicsEntry_ { ...@@ -60,13 +56,15 @@ const struct IceIntrinsicsEntry_ {
AtomicCmpxchgInit(IceType_i64, "i64"), AtomicCmpxchgInit(IceType_i64, "i64"),
#undef AtomicCmpxchgInit #undef AtomicCmpxchgInit
{{INTRIN(AtomicFence, SideEffects_T, ReturnsTwice_F), {{INTRIN(AtomicFence, SideEffects_T, ReturnsTwice_F, MemoryWrite_T),
{IceType_void, IceType_i32}, {IceType_void, IceType_i32},
2}, 2},
"llvm.nacl.atomic.fence"}, "llvm.nacl.atomic.fence"},
{{INTRIN(AtomicFenceAll, SideEffects_T, ReturnsTwice_F), {IceType_void}, 1}, {{INTRIN(AtomicFenceAll, SideEffects_T, ReturnsTwice_F, MemoryWrite_T),
{IceType_void},
1},
"llvm.nacl.atomic.fence.all"}, "llvm.nacl.atomic.fence.all"},
{{INTRIN(AtomicIsLockFree, SideEffects_F, ReturnsTwice_F), {{INTRIN(AtomicIsLockFree, SideEffects_F, ReturnsTwice_F, MemoryWrite_F),
{IceType_i1, IceType_i32, IceType_i32}, {IceType_i1, IceType_i32, IceType_i32},
3}, 3},
"llvm.nacl.atomic.is.lock.free"}, "llvm.nacl.atomic.is.lock.free"},
...@@ -74,7 +72,7 @@ const struct IceIntrinsicsEntry_ { ...@@ -74,7 +72,7 @@ const struct IceIntrinsicsEntry_ {
#define AtomicLoadInit(Overload, NameSuffix) \ #define AtomicLoadInit(Overload, NameSuffix) \
{ \ { \
{ \ { \
INTRIN(AtomicLoad, SideEffects_T, ReturnsTwice_F), \ INTRIN(AtomicLoad, SideEffects_T, ReturnsTwice_F, MemoryWrite_T), \
{Overload, IceType_i32, IceType_i32}, 3 \ {Overload, IceType_i32, IceType_i32}, 3 \
} \ } \
, "llvm.nacl.atomic.load." NameSuffix \ , "llvm.nacl.atomic.load." NameSuffix \
...@@ -88,7 +86,7 @@ const struct IceIntrinsicsEntry_ { ...@@ -88,7 +86,7 @@ const struct IceIntrinsicsEntry_ {
#define AtomicRMWInit(Overload, NameSuffix) \ #define AtomicRMWInit(Overload, NameSuffix) \
{ \ { \
{ \ { \
INTRIN(AtomicRMW, SideEffects_T, ReturnsTwice_F) \ INTRIN(AtomicRMW, SideEffects_T, ReturnsTwice_F, MemoryWrite_T) \
, {Overload, IceType_i32, IceType_i32, Overload, IceType_i32}, 5 \ , {Overload, IceType_i32, IceType_i32, Overload, IceType_i32}, 5 \
} \ } \
, "llvm.nacl.atomic.rmw." NameSuffix \ , "llvm.nacl.atomic.rmw." NameSuffix \
...@@ -102,7 +100,7 @@ const struct IceIntrinsicsEntry_ { ...@@ -102,7 +100,7 @@ const struct IceIntrinsicsEntry_ {
#define AtomicStoreInit(Overload, NameSuffix) \ #define AtomicStoreInit(Overload, NameSuffix) \
{ \ { \
{ \ { \
INTRIN(AtomicStore, SideEffects_T, ReturnsTwice_F) \ INTRIN(AtomicStore, SideEffects_T, ReturnsTwice_F, MemoryWrite_T) \
, {IceType_void, Overload, IceType_i32, IceType_i32}, 4 \ , {IceType_void, Overload, IceType_i32, IceType_i32}, 4 \
} \ } \
, "llvm.nacl.atomic.store." NameSuffix \ , "llvm.nacl.atomic.store." NameSuffix \
...@@ -116,7 +114,7 @@ const struct IceIntrinsicsEntry_ { ...@@ -116,7 +114,7 @@ const struct IceIntrinsicsEntry_ {
#define BswapInit(Overload, NameSuffix) \ #define BswapInit(Overload, NameSuffix) \
{ \ { \
{ \ { \
INTRIN(Bswap, SideEffects_F, ReturnsTwice_F) \ INTRIN(Bswap, SideEffects_F, ReturnsTwice_F, MemoryWrite_F) \
, {Overload, Overload}, 2 \ , {Overload, Overload}, 2 \
} \ } \
, "llvm.bswap." NameSuffix \ , "llvm.bswap." NameSuffix \
...@@ -129,7 +127,7 @@ const struct IceIntrinsicsEntry_ { ...@@ -129,7 +127,7 @@ const struct IceIntrinsicsEntry_ {
#define CtlzInit(Overload, NameSuffix) \ #define CtlzInit(Overload, NameSuffix) \
{ \ { \
{ \ { \
INTRIN(Ctlz, SideEffects_F, ReturnsTwice_F) \ INTRIN(Ctlz, SideEffects_F, ReturnsTwice_F, MemoryWrite_F) \
, {Overload, Overload, IceType_i1}, 3 \ , {Overload, Overload, IceType_i1}, 3 \
} \ } \
, "llvm.ctlz." NameSuffix \ , "llvm.ctlz." NameSuffix \
...@@ -141,7 +139,7 @@ const struct IceIntrinsicsEntry_ { ...@@ -141,7 +139,7 @@ const struct IceIntrinsicsEntry_ {
#define CtpopInit(Overload, NameSuffix) \ #define CtpopInit(Overload, NameSuffix) \
{ \ { \
{ \ { \
INTRIN(Ctpop, SideEffects_F, ReturnsTwice_F) \ INTRIN(Ctpop, SideEffects_F, ReturnsTwice_F, MemoryWrite_F) \
, {Overload, Overload}, 2 \ , {Overload, Overload}, 2 \
} \ } \
, "llvm.ctpop." NameSuffix \ , "llvm.ctpop." NameSuffix \
...@@ -153,7 +151,7 @@ const struct IceIntrinsicsEntry_ { ...@@ -153,7 +151,7 @@ const struct IceIntrinsicsEntry_ {
#define CttzInit(Overload, NameSuffix) \ #define CttzInit(Overload, NameSuffix) \
{ \ { \
{ \ { \
INTRIN(Cttz, SideEffects_F, ReturnsTwice_F) \ INTRIN(Cttz, SideEffects_F, ReturnsTwice_F, MemoryWrite_F) \
, {Overload, Overload, IceType_i1}, 3 \ , {Overload, Overload, IceType_i1}, 3 \
} \ } \
, "llvm.cttz." NameSuffix \ , "llvm.cttz." NameSuffix \
...@@ -164,7 +162,10 @@ const struct IceIntrinsicsEntry_ { ...@@ -164,7 +162,10 @@ const struct IceIntrinsicsEntry_ {
#define FabsInit(Overload, NameSuffix) \ #define FabsInit(Overload, NameSuffix) \
{ \ { \
{ INTRIN(Fabs, SideEffects_F, ReturnsTwice_F), {Overload, Overload}, 2 } \ { \
INTRIN(Fabs, SideEffects_F, ReturnsTwice_F, MemoryWrite_F), \
{Overload, Overload}, 2 \
} \
, "llvm.fabs." NameSuffix \ , "llvm.fabs." NameSuffix \
} }
FabsInit(IceType_f32, "f32"), FabsInit(IceType_f32, "f32"),
...@@ -172,48 +173,57 @@ const struct IceIntrinsicsEntry_ { ...@@ -172,48 +173,57 @@ const struct IceIntrinsicsEntry_ {
FabsInit(IceType_v4f32, "v4f32"), FabsInit(IceType_v4f32, "v4f32"),
#undef FabsInit #undef FabsInit
{{INTRIN(Longjmp, SideEffects_T, ReturnsTwice_F), {{INTRIN(Longjmp, SideEffects_T, ReturnsTwice_F, MemoryWrite_F),
{IceType_void, IceType_i32, IceType_i32}, {IceType_void, IceType_i32, IceType_i32},
3}, 3},
"llvm.nacl.longjmp"}, "llvm.nacl.longjmp"},
{{INTRIN(Memcpy, SideEffects_T, ReturnsTwice_F), {{INTRIN(Memcpy, SideEffects_T, ReturnsTwice_F, MemoryWrite_T),
{IceType_void, IceType_i32, IceType_i32, IceType_i32, IceType_i32, {IceType_void, IceType_i32, IceType_i32, IceType_i32, IceType_i32,
IceType_i1}, IceType_i1},
6}, 6},
"llvm.memcpy.p0i8.p0i8.i32"}, "llvm.memcpy.p0i8.p0i8.i32"},
{{INTRIN(Memmove, SideEffects_T, ReturnsTwice_F), {{INTRIN(Memmove, SideEffects_T, ReturnsTwice_F, MemoryWrite_T),
{IceType_void, IceType_i32, IceType_i32, IceType_i32, IceType_i32, {IceType_void, IceType_i32, IceType_i32, IceType_i32, IceType_i32,
IceType_i1}, IceType_i1},
6}, 6},
"llvm.memmove.p0i8.p0i8.i32"}, "llvm.memmove.p0i8.p0i8.i32"},
{{INTRIN(Memset, SideEffects_T, ReturnsTwice_F), {{INTRIN(Memset, SideEffects_T, ReturnsTwice_F, MemoryWrite_T),
{IceType_void, IceType_i32, IceType_i8, IceType_i32, IceType_i32, {IceType_void, IceType_i32, IceType_i8, IceType_i32, IceType_i32,
IceType_i1}, IceType_i1},
6}, 6},
"llvm.memset.p0i8.i32"}, "llvm.memset.p0i8.i32"},
{{INTRIN(NaClReadTP, SideEffects_F, ReturnsTwice_F), {IceType_i32}, 1}, {{INTRIN(NaClReadTP, SideEffects_F, ReturnsTwice_F, MemoryWrite_F),
{IceType_i32},
1},
"llvm.nacl.read.tp"}, "llvm.nacl.read.tp"},
{{INTRIN(Setjmp, SideEffects_T, ReturnsTwice_T), {{INTRIN(Setjmp, SideEffects_T, ReturnsTwice_T, MemoryWrite_T),
{IceType_i32, IceType_i32}, {IceType_i32, IceType_i32},
2}, 2},
"llvm.nacl.setjmp"}, "llvm.nacl.setjmp"},
#define SqrtInit(Overload, NameSuffix) \ #define SqrtInit(Overload, NameSuffix) \
{ \ { \
{ INTRIN(Sqrt, SideEffects_F, ReturnsTwice_F), {Overload, Overload}, 2 } \ { \
INTRIN(Sqrt, SideEffects_F, ReturnsTwice_F, MemoryWrite_F), \
{Overload, Overload}, 2 \
} \
, "llvm.sqrt." NameSuffix \ , "llvm.sqrt." NameSuffix \
} }
SqrtInit(IceType_f32, "f32"), SqrtInit(IceType_f32, "f32"),
SqrtInit(IceType_f64, "f64"), SqrtInit(IceType_f64, "f64"),
#undef SqrtInit #undef SqrtInit
{{INTRIN(Stacksave, SideEffects_T, ReturnsTwice_F), {IceType_i32}, 1}, {{INTRIN(Stacksave, SideEffects_T, ReturnsTwice_F, MemoryWrite_F),
{IceType_i32},
1},
"llvm.stacksave"}, "llvm.stacksave"},
{{INTRIN(Stackrestore, SideEffects_T, ReturnsTwice_F), {{INTRIN(Stackrestore, SideEffects_T, ReturnsTwice_F, MemoryWrite_F),
{IceType_void, IceType_i32}, {IceType_void, IceType_i32},
2}, 2},
"llvm.stackrestore"}, "llvm.stackrestore"},
{{INTRIN(Trap, SideEffects_T, ReturnsTwice_F), {IceType_void}, 1}, {{INTRIN(Trap, SideEffects_T, ReturnsTwice_F, MemoryWrite_F),
{IceType_void},
1},
"llvm.trap"}}; "llvm.trap"}};
const size_t IceIntrinsicsTableSize = llvm::array_lengthof(IceIntrinsicsTable); const size_t IceIntrinsicsTableSize = llvm::array_lengthof(IceIntrinsicsTable);
......
src/IceIntrinsics.h
...@@ -106,15 +106,19 @@ public: ...@@ -106,15 +106,19 @@ public:
enum ReturnsTwice { ReturnsTwice_F = 0, ReturnsTwice_T = 1 }; enum ReturnsTwice { ReturnsTwice_F = 0, ReturnsTwice_T = 1 };
enum MemoryWrite { MemoryWrite_F = 0, MemoryWrite_T = 1 };
/// Basic attributes related to each intrinsic, that are relevant to code /// Basic attributes related to each intrinsic, that are relevant to code
/// generation. Perhaps the attributes representation can be shared with /// generation. Perhaps the attributes representation can be shared with
/// general function calls, but PNaCl currently strips all attributes from /// general function calls, but PNaCl currently strips all attributes from
/// functions. /// functions.
struct IntrinsicInfo { struct IntrinsicInfo {
enum IntrinsicID ID : 30; enum IntrinsicID ID : 29;
enum SideEffects HasSideEffects : 1; enum SideEffects HasSideEffects : 1;
enum ReturnsTwice ReturnsTwice : 1; enum ReturnsTwice ReturnsTwice : 1;
enum MemoryWrite IsMemoryWrite : 1;
}; };
static_assert(sizeof(IntrinsicInfo) == 4, "IntrinsicInfo should be 32 bits");
/// The types of validation values for FullIntrinsicInfo.validateCall. /// The types of validation values for FullIntrinsicInfo.validateCall.
enum ValidateCallValue { enum ValidateCallValue {
......
src/IceTargetLoweringX86BaseImpl.h
...@@ -151,6 +151,7 @@ private: ...@@ -151,6 +151,7 @@ private:
return Element != Producers.end() && Element->second.Instr != nullptr; return Element != Producers.end() && Element->second.Instr != nullptr;
} }
void setInvalid(SizeT VarNum) { Producers[VarNum].Instr = nullptr; } void setInvalid(SizeT VarNum) { Producers[VarNum].Instr = nullptr; }
void invalidateProducersOnStore(const Inst *Instr);
/// Producers maps Variable::Number to a BoolFoldingEntry. /// Producers maps Variable::Number to a BoolFoldingEntry.
CfgUnorderedMap<SizeT, BoolFoldingEntry<Traits>> Producers; CfgUnorderedMap<SizeT, BoolFoldingEntry<Traits>> Producers;
}; };
...@@ -252,10 +253,12 @@ bool BoolFolding<Traits>::isValidFolding( ...@@ -252,10 +253,12 @@ bool BoolFolding<Traits>::isValidFolding(
template <typename Traits> void BoolFolding<Traits>::init(CfgNode *Node) { template <typename Traits> void BoolFolding<Traits>::init(CfgNode *Node) {
Producers.clear(); Producers.clear();
for (Inst &Instr : Node->getInsts()) { for (Inst &Instr : Node->getInsts()) {
if (Instr.isDeleted())
continue;
invalidateProducersOnStore(&Instr);
// Check whether Instr is a valid producer. // Check whether Instr is a valid producer.
Variable *Var = Instr.getDest(); Variable *Var = Instr.getDest();
if (!Instr.isDeleted() // only consider non-deleted instructions if (Var // only consider instructions with an actual dest var
&& Var // only instructions with an actual dest var
&& Var->getType() == IceType_i1 // only bool-type dest vars && Var->getType() == IceType_i1 // only bool-type dest vars
&& getProducerKind(&Instr) != PK_None) { // white-listed instructions && getProducerKind(&Instr) != PK_None) { // white-listed instructions
Producers[Var->getIndex()] = BoolFoldingEntry<Traits>(&Instr); Producers[Var->getIndex()] = BoolFoldingEntry<Traits>(&Instr);
...@@ -338,6 +341,43 @@ void BoolFolding<Traits>::dump(const Cfg *Func) const { ...@@ -338,6 +341,43 @@ void BoolFolding<Traits>::dump(const Cfg *Func) const {
} }
} }
/// If the given instruction has potential memory side effects (e.g. store, rmw,
/// or a call instruction with potential memory side effects), then we must not
/// allow a pre-store Producer instruction with memory operands to be folded
/// into a post-store Consumer instruction. If this is detected, the Producer
/// is invalidated.
///
/// We use the Producer's IsLiveOut field to determine whether any potential
/// Consumers come after this store instruction. The IsLiveOut field is
/// initialized to true, and BoolFolding::init() sets IsLiveOut to false when it
/// sees the variable's definitive last use (indicating the variable is not in
/// the node's live-out set). Thus if we see here that IsLiveOut is false, we
/// know that there can be no consumers after the store, and therefore we know
/// the folding is safe despite the store instruction.
template <typename Traits>
void BoolFolding<Traits>::invalidateProducersOnStore(const Inst *Instr) {
if (!Instr->isMemoryWrite())
return;
for (auto &ProducerPair : Producers) {
if (!ProducerPair.second.IsLiveOut)
continue;
Inst *PInst = ProducerPair.second.Instr;
if (PInst == nullptr)
continue;
bool HasMemOperand = false;
const SizeT SrcSize = PInst->getSrcSize();
for (SizeT I = 0; I < SrcSize; ++I) {
if (llvm::isa<typename Traits::X86OperandMem>(PInst->getSrc(I))) {
HasMemOperand = true;
break;
}
}
if (!HasMemOperand)
continue;
setInvalid(ProducerPair.first);
}
}
template <typename TraitsType> template <typename TraitsType>
void TargetX86Base<TraitsType>::initNodeForLowering(CfgNode *Node) { void TargetX86Base<TraitsType>::initNodeForLowering(CfgNode *Node) {
FoldingInfo.init(Node); FoldingInfo.init(Node);
......
tests_lit/llvm2ice_tests/bool-folding.ll
...@@ -269,6 +269,58 @@ next: ...@@ -269,6 +269,58 @@ next:
; ARM32: movne ; ARM32: movne
; ARM32: bx lr ; ARM32: bx lr
; Cmp/branch non-folding due to load folding and intervening store.
define internal i32 @no_fold_cmp_br_store(i32 %arg2, i32 %argaddr) {
entry:
%addr = inttoptr i32 %argaddr to i32*
%arg1 = load i32, i32* %addr, align 1
%cmp1 = icmp slt i32 %arg1, %arg2
store i32 1, i32* %addr, align 1
br i1 %cmp1, label %branch1, label %branch2
branch1:
ret i32 1
branch2:
ret i32 2
}
; CHECK-LABEL: no_fold_cmp_br_store
; CHECK: cmp
; CHECK: set
; CHECK: cmp
; Cmp/select non-folding due to load folding and intervening store.
define internal i32 @no_fold_cmp_select_store(i32 %arg1, i32 %argaddr) {
entry:
%addr = inttoptr i32 %argaddr to i32*
%arg2 = load i32, i32* %addr, align 1
%cmp1 = icmp slt i32 %arg1, %arg2
store i32 1, i32* %addr, align 1
%result = select i1 %cmp1, i32 %arg1, i32 %argaddr
ret i32 %result
}
; CHECK-LABEL: no_fold_cmp_select_store
; CHECK: cmp
; CHECK: setl
; CHECK: mov DWORD PTR
; CHECK: cmp
; CHECK: cmovne
; Cmp/select folding due to load folding and non-intervening store.
define internal i32 @fold_cmp_select_store(i32 %arg1, i32 %argaddr) {
entry:
%addr = inttoptr i32 %argaddr to i32*
%arg2 = load i32, i32* %addr, align 1
%cmp1 = icmp slt i32 %arg1, %arg2
%result = select i1 %cmp1, i32 %arg1, i32 %argaddr
store i32 1, i32* %addr, align 1
ret i32 %result
}
; CHECK-LABEL: fold_cmp_select_store
; CHECK: cmp {{.*}},DWORD PTR
; CHECK: cmovl
; Cmp/multi-select non-folding because of extra non-whitelisted uses. ; Cmp/multi-select non-folding because of extra non-whitelisted uses.
define internal i32 @no_fold_cmp_select_multi_non_whitelist(i32 %arg1, define internal i32 @no_fold_cmp_select_multi_non_whitelist(i32 %arg1,
i32 %arg2) { i32 %arg2) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment