Skip to content

S
swiftshader
Commit 1bdb7352 by Jim Stichnoth
Options

Subzero: Reduce copying of Liveness bitvectors.

There were a lot of unnecessary copying and resizing and sloppiness in the use of BitVector for liveness analysis. This tries to reduce the amount of copying and associated memory allocation. Also, works around a ConstantRelocatable hashing problem that was causing a huge slowdown in MINIMAL mode for the vector_shuffle scons test. BUG= none R=jpp@chromium.org Review URL: https://codereview.chromium.org/1746613002 .
parent fc707ffe
Showing with 93 additions and 58 deletions
src/IceBitVector.h
......@@ -308,7 +308,7 @@ public:
explicit BitVector(unsigned s, bool t = false, Allocator A = Allocator())
: Size(s), Alloc(std::move(A)) {
Capacity = NumBitWords(s);
Bits = Alloc.allocate(Capacity * sizeof(BitWord));
Bits = Alloc.allocate(Capacity);
init_words(Bits, Capacity, t);
if (t)
clear_unused_bits();
......@@ -323,7 +323,7 @@ public:
}
Capacity = NumBitWords(RHS.size());
Bits = Alloc.allocate(Capacity * sizeof(BitWord));
Bits = Alloc.allocate(Capacity);
std::memcpy(Bits, RHS.Bits, Capacity * sizeof(BitWord));
}
......@@ -335,7 +335,7 @@ public:
~BitVector() {
if (Bits != nullptr) {
Alloc.deallocate(Bits, Capacity * sizeof(BitWord));
Alloc.deallocate(Bits, Capacity);
}
}
......@@ -655,15 +655,21 @@ public:
return *this;
}
// Currently, BitVector is only used by liveness analysis. With the
// following assert, we make sure BitVectors grow in a single step from 0 to
// their final capacity, rather than growing slowly and "leaking" memory in
// the process.
assert(Capacity == 0);
// Grow the bitvector to have enough elements.
const auto OldCapacity = Capacity;
Capacity = RHSWords;
assert(Capacity > 0 && "negative capacity?");
BitWord *NewBits = Alloc.allocate(Capacity * sizeof(BitWord));
BitWord *NewBits = Alloc.allocate(Capacity);
std::memcpy(NewBits, RHS.Bits, Capacity * sizeof(BitWord));
// Destroy the old bits.
Alloc.deallocate(Bits, OldCapacity * sizeof(BitWord));
Alloc.deallocate(Bits, OldCapacity);
Bits = NewBits;
return *this;
......@@ -673,7 +679,7 @@ public:
if (this == &RHS)
return *this;
Alloc.deallocate(Bits, Capacity * sizeof(BitWord));
Alloc.deallocate(Bits, Capacity);
Bits = RHS.Bits;
Size = RHS.Size;
Capacity = RHS.Capacity;
......@@ -755,9 +761,9 @@ private:
const auto OldCapacity = Capacity;
Capacity = std::max(NumBitWords(NewSize), Capacity * 2);
assert(Capacity > 0 && "realloc-ing zero space");
auto *NewBits = Alloc.allocate(Capacity * sizeof(BitWord));
auto *NewBits = Alloc.allocate(Capacity);
std::memcpy(Bits, NewBits, OldCapacity * sizeof(BitWord));
Alloc.deallocate(Bits, OldCapacity * sizeof(BitWord));
Alloc.deallocate(Bits, OldCapacity);
Bits = NewBits;
clear_unused_bits();
......
src/IceCfg.cpp
......@@ -1076,6 +1076,14 @@ void Cfg::emitIAS() {
emitJumpTables();
}
size_t Cfg::getTotalMemoryMB() {
constexpr size_t OneMB = 1024 * 1024;
using ArbitraryType = int;
// CfgLocalAllocator draws from the same memory pool regardless of allocated
// object type, so pick an arbitrary type for the template parameter.
return CfgLocalAllocator<ArbitraryType>().current()->getTotalMemory() / OneMB;
}
// Dumps the IR with an optional introductory message.
void Cfg::dump(const IceString &Message) {
if (!BuildDefs::dump())
......@@ -1087,10 +1095,7 @@ void Cfg::dump(const IceString &Message) {
if (!Message.empty())
Str << "================ " << Message << " ================\n";
if (isVerbose(IceV_Mem)) {
constexpr size_t OneMB = 1024 * 1024;
Str << "Memory size = "
<< (CfgLocalAllocator<int>().current()->getTotalMemory() / OneMB)
<< " MB\n";
Str << "Memory size = " << getTotalMemoryMB() << " MB\n";
}
setCurrentNode(getEntryNode());
// Print function name+args
......
src/IceCfg.h
......@@ -219,6 +219,10 @@ public:
const CfgNode *getCurrentNode() const { return CurrentNode; }
/// @}
/// Get the total amount of memory held by the per-Cfg allocator. This is
/// mostly meant for use inside a debugger.
static size_t getTotalMemoryMB();
void emit();
void emitIAS();
static void emitTextHeader(const IceString &MangledName, GlobalContext *Ctx,
......
src/IceCfgNode.cpp
......@@ -626,8 +626,11 @@ void CfgNode::livenessLightweight() {
// liveness changed from before, false if it stayed the same. (If it changes,
// the node's predecessors need to be processed again.)
bool CfgNode::liveness(Liveness *Liveness) {
SizeT NumVars = Liveness->getNumVarsInNode(this);
LivenessBV Live(NumVars);
const SizeT NumVars = Liveness->getNumVarsInNode(this);
const SizeT NumGlobalVars = Liveness->getNumGlobalVars();
LivenessBV &Live = Liveness->getScratchBV();
Live.clear();
LiveBeginEndMap *LiveBegin = nullptr;
LiveBeginEndMap *LiveEnd = nullptr;
// Mark the beginning and ending of each variable's live range with the
......@@ -642,19 +645,25 @@ bool CfgNode::liveness(Liveness *Liveness) {
LiveBegin->reserve(getInstCountEstimate());
LiveEnd->reserve(getInstCountEstimate());
}
// Initialize Live to be the union of all successors' LiveIn.
for (CfgNode *Succ : OutEdges) {
Live |= Liveness->getLiveIn(Succ);
const LivenessBV &LiveIn = Liveness->getLiveIn(Succ);
assert(LiveIn.empty() || LiveIn.size() == NumGlobalVars);
Live |= LiveIn;
// Mark corresponding argument of phis in successor as live.
for (Inst &I : Succ->Phis) {
if (I.isDeleted())
continue;
auto *Phi = llvm::dyn_cast<InstPhi>(&I);
auto *Phi = llvm::cast<InstPhi>(&I);
Phi->livenessPhiOperand(Live, this, Liveness);
}
}
assert(Live.empty() || Live.size() == NumGlobalVars);
Liveness->getLiveOut(this) = Live;
// Expand Live so it can hold locals in addition to globals.
Live.resize(NumVars);
// Process regular instructions in reverse order.
for (Inst &I : reverse_range(Insts)) {
if (I.isDeleted())
......@@ -676,20 +685,17 @@ bool CfgNode::liveness(Liveness *Liveness) {
// When using the sparse representation, after traversing the instructions in
// the block, the Live bitvector should only contain set bits for global
// variables upon block entry. We validate this by shrinking the Live vector
// and then testing it against the pre-shrunk version. (The shrinking is
// required, but the validation is not.)
LivenessBV LiveOrig = Live;
Live.resize(Liveness->getNumGlobalVars());
if (Live != LiveOrig) {
// variables upon block entry. We validate this by testing the upper bits of
// the Live bitvector.
if (Live.find_next(NumGlobalVars) != -1) {
if (BuildDefs::dump()) {
// This is a fatal liveness consistency error. Print some diagnostics and
// abort.
Ostream &Str = Func->getContext()->getStrDump();
Func->resetCurrentNode();
Str << "LiveOrig-Live =";
for (SizeT i = Live.size(); i < LiveOrig.size(); ++i) {
if (LiveOrig.test(i)) {
Str << "Invalid Live =";
for (SizeT i = NumGlobalVars; i < Live.size(); ++i) {
if (Live.test(i)) {
Str << " ";
Liveness->getVariable(i, this)->dump(Func);
}
......@@ -698,9 +704,12 @@ bool CfgNode::liveness(Liveness *Liveness) {
}
llvm::report_fatal_error("Fatal inconsistency in liveness analysis");
}
// Now truncate Live to prevent LiveIn from growing.
Live.resize(NumGlobalVars);
bool Changed = false;
LivenessBV &LiveIn = Liveness->getLiveIn(this);
assert(LiveIn.empty() || LiveIn.size() == NumGlobalVars);
// Add in current LiveIn
Live |= LiveIn;
// Check result, set LiveIn=Live
......@@ -778,9 +787,9 @@ void CfgNode::livenessAddIntervals(Liveness *Liveness, InstNumberT FirstInstNum,
InstNumberT LastInstNum) {
TimerMarker T1(TimerStack::TT_liveRange, Func);
SizeT NumVars = Liveness->getNumVarsInNode(this);
LivenessBV &LiveIn = Liveness->getLiveIn(this);
LivenessBV &LiveOut = Liveness->getLiveOut(this);
const SizeT NumVars = Liveness->getNumVarsInNode(this);
const LivenessBV &LiveIn = Liveness->getLiveIn(this);
const LivenessBV &LiveOut = Liveness->getLiveOut(this);
LiveBeginEndMap &MapBegin = *Liveness->getLiveBegin(this);
LiveBeginEndMap &MapEnd = *Liveness->getLiveEnd(this);
std::sort(MapBegin.begin(), MapBegin.end());
......@@ -791,7 +800,8 @@ void CfgNode::livenessAddIntervals(Liveness *Liveness, InstNumberT FirstInstNum,
return;
}
LivenessBV LiveInAndOut = LiveIn;
LivenessBV &LiveInAndOut = Liveness->getScratchBV();
LiveInAndOut = LiveIn;
LiveInAndOut &= LiveOut;
// Iterate in parallel across the sorted MapBegin[] and MapEnd[].
......@@ -1350,23 +1360,23 @@ void CfgNode::dump(Cfg *Func) const {
Str << "\n";
}
// Dump the live-in variables.
LivenessBV LiveIn;
if (Liveness)
LiveIn = Liveness->getLiveIn(this);
if (Func->isVerbose(IceV_Liveness) && !LiveIn.empty()) {
Str << " // LiveIn:";
for (SizeT i = 0; i < LiveIn.size(); ++i) {
if (LiveIn[i]) {
Variable *Var = Liveness->getVariable(i, this);
Str << " %" << Var->getName(Func);
if (Func->isVerbose(IceV_RegOrigins) && Var->hasReg()) {
Str << ":"
<< Func->getTarget()->getRegName(Var->getRegNum(),
Var->getType());
if (Func->isVerbose(IceV_Liveness)) {
if (Liveness != nullptr && !Liveness->getLiveIn(this).empty()) {
const LivenessBV &LiveIn = Liveness->getLiveIn(this);
Str << " // LiveIn:";
for (SizeT i = 0; i < LiveIn.size(); ++i) {
if (LiveIn[i]) {
Variable *Var = Liveness->getVariable(i, this);
Str << " %" << Var->getName(Func);
if (Func->isVerbose(IceV_RegOrigins) && Var->hasReg()) {
Str << ":"
<< Func->getTarget()->getRegName(Var->getRegNum(),
Var->getType());
}
}
}
Str << "\n";
}
Str << "\n";
}
// Dump each instruction.
if (Func->isVerbose(IceV_Instructions)) {
......@@ -1376,23 +1386,23 @@ void CfgNode::dump(Cfg *Func) const {
I.dumpDecorated(Func);
}
// Dump the live-out variables.
LivenessBV LiveOut;
if (Liveness)
LiveOut = Liveness->getLiveOut(this);
if (Func->isVerbose(IceV_Liveness) && !LiveOut.empty()) {
Str << " // LiveOut:";
for (SizeT i = 0; i < LiveOut.size(); ++i) {
if (LiveOut[i]) {
Variable *Var = Liveness->getVariable(i, this);
Str << " %" << Var->getName(Func);
if (Func->isVerbose(IceV_RegOrigins) && Var->hasReg()) {
Str << ":"
<< Func->getTarget()->getRegName(Var->getRegNum(),
Var->getType());
if (Func->isVerbose(IceV_Liveness)) {
if (Liveness != nullptr && !Liveness->getLiveOut(this).empty()) {
const LivenessBV &LiveOut = Liveness->getLiveOut(this);
Str << " // LiveOut:";
for (SizeT i = 0; i < LiveOut.size(); ++i) {
if (LiveOut[i]) {
Variable *Var = Liveness->getVariable(i, this);
Str << " %" << Var->getName(Func);
if (Func->isVerbose(IceV_RegOrigins) && Var->hasReg()) {
Str << ":"
<< Func->getTarget()->getRegName(Var->getRegNum(),
Var->getType());
}
}
}
Str << "\n";
}
Str << "\n";
}
// Dump list of successor nodes.
if (Func->isVerbose(IceV_Succs)) {
......
src/IceInstX86BaseImpl.h
......@@ -98,8 +98,11 @@ InstImpl<TraitsType>::InstX86Label::InstX86Label(Cfg *Func,
template <typename TraitsType>
IceString InstImpl<TraitsType>::InstX86Label::getName(const Cfg *Func) const {
// TODO(stichnot): Returning an empty string in a non-DUMP build can cause a
// huge degradation in ConstantRelocatable hashing. Investigate and fix, but
// for now return something reasonably unique.
if (!BuildDefs::dump())
return IceString();
return Func->getFunctionName() + std::to_string(Number);
return ".L" + Func->getFunctionName() + "$local$__" + std::to_string(Number);
}
......
src/IceLiveness.cpp
......@@ -106,6 +106,7 @@ void Liveness::initInternal(NodeList::const_iterator FirstNode,
RangeMask[VarIndex] = false;
}
SizeT MaxLocals = 0;
// Process each node.
for (auto I = FirstNode, E = Func->getNodes().end(); I != E; ++I) {
LivenessNode &Node = Nodes[(*I)->getIndex()];
......@@ -113,7 +114,9 @@ void Liveness::initInternal(NodeList::const_iterator FirstNode,
Node.LiveIn.resize(NumGlobals);
Node.LiveOut.resize(NumGlobals);
// LiveBegin and LiveEnd are reinitialized before each pass over the block.
MaxLocals = std::max(MaxLocals, Node.NumLocals);
}
ScratchBV.reserve(NumGlobals + MaxLocals);
}
void Liveness::init() {
......
src/IceLiveness.h
......@@ -85,6 +85,7 @@ public:
resize(Index);
return Nodes[Index].LiveOut;
}
LivenessBV &getScratchBV() { return ScratchBV; }
LiveBeginEndMap *getLiveBegin(const CfgNode *Node) {
SizeT Index = Node->getIndex();
resize(Index);
......@@ -119,6 +120,9 @@ private:
/// RangeMask[Variable::Number] indicates whether we want to track that
/// Variable's live range.
LivenessBV RangeMask;
/// ScratchBV is a bitvector that can be reused across CfgNode passes, to
/// avoid having to allocate/deallocate memory so frequently.
LivenessBV ScratchBV;
};
} // end of namespace Ice
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment