Skip to content

S
swiftshader
Commit d24cfda1 by Andrew Scull
Options

Refactor LinearScan::scan from one huge function into smaller functions.

BUG= R=jvoung@chromium.org, stichnot@chromium.org Review URL: https://codereview.chromium.org/1310833003.
parent 0042fea3
Showing with 438 additions and 407 deletions
src/IceRegAlloc.cpp
...@@ -8,9 +8,8 @@ ...@@ -8,9 +8,8 @@
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
/// ///
/// \file /// \file
/// This file implements the LinearScan class, which performs the /// This file implements the LinearScan class, which performs the linear-scan
/// linear-scan register allocation after liveness analysis has been /// register allocation after liveness analysis has been performed.
/// performed.
/// ///
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
...@@ -26,16 +25,12 @@ namespace Ice { ...@@ -26,16 +25,12 @@ namespace Ice {
namespace { namespace {
// TODO(stichnot): Statically choose the size based on the target
// being compiled.
constexpr size_t REGS_SIZE = 32;
// Returns true if Var has any definitions within Item's live range. // Returns true if Var has any definitions within Item's live range.
// TODO(stichnot): Consider trimming the Definitions list similar to // TODO(stichnot): Consider trimming the Definitions list similar to how the
// how the live ranges are trimmed, since all the overlapsDefs() tests // live ranges are trimmed, since all the overlapsDefs() tests are whether some
// are whether some variable's definitions overlap Cur, and trimming // variable's definitions overlap Cur, and trimming is with respect Cur.start.
// is with respect Cur.start. Initial tests show no measurable // Initial tests show no measurable performance difference, so we'll keep the
// performance difference, so we'll keep the code simple for now. // code simple for now.
bool overlapsDefs(const Cfg *Func, const Variable *Item, const Variable *Var) { bool overlapsDefs(const Cfg *Func, const Variable *Item, const Variable *Var) {
constexpr bool UseTrimmed = true; constexpr bool UseTrimmed = true;
VariablesMetadata *VMetadata = Func->getVMetadata(); VariablesMetadata *VMetadata = Func->getVMetadata();
...@@ -82,8 +77,12 @@ void dumpLiveRange(const Variable *Var, const Cfg *Func) { ...@@ -82,8 +77,12 @@ void dumpLiveRange(const Variable *Var, const Cfg *Func) {
} // end of anonymous namespace } // end of anonymous namespace
// Prepare for full register allocation of all variables. We depend LinearScan::LinearScan(Cfg *Func)
// on liveness analysis to have calculated live ranges. : Func(Func), Ctx(Func->getContext()),
Verbose(BuildDefs::dump() && Func->isVerbose(IceV_LinearScan)) {}
// Prepare for full register allocation of all variables. We depend on
// liveness analysis to have calculated live ranges.
void LinearScan::initForGlobal() { void LinearScan::initForGlobal() {
TimerMarker T(TimerStack::TT_initUnhandled, Func); TimerMarker T(TimerStack::TT_initUnhandled, Func);
FindPreference = true; FindPreference = true;
...@@ -96,15 +95,14 @@ void LinearScan::initForGlobal() { ...@@ -96,15 +95,14 @@ void LinearScan::initForGlobal() {
const VarList &Vars = Func->getVariables(); const VarList &Vars = Func->getVariables();
Unhandled.reserve(Vars.size()); Unhandled.reserve(Vars.size());
UnhandledPrecolored.reserve(Vars.size()); UnhandledPrecolored.reserve(Vars.size());
// Gather the live ranges of all variables and add them to the // Gather the live ranges of all variables and add them to the Unhandled set.
// Unhandled set.
for (Variable *Var : Vars) { for (Variable *Var : Vars) {
// Explicitly don't consider zero-weight variables, which are // Explicitly don't consider zero-weight variables, which are meant to be
// meant to be spill slots. // spill slots.
if (Var->getWeight().isZero()) if (Var->getWeight().isZero())
continue; continue;
// Don't bother if the variable has a null live range, which means // Don't bother if the variable has a null live range, which means it was
// it was never referenced. // never referenced.
if (Var->getLiveRange().isEmpty()) if (Var->getLiveRange().isEmpty())
continue; continue;
Var->untrimLiveRange(); Var->untrimLiveRange();
...@@ -134,33 +132,30 @@ void LinearScan::initForGlobal() { ...@@ -134,33 +132,30 @@ void LinearScan::initForGlobal() {
} }
// Prepare for very simple register allocation of only infinite-weight // Prepare for very simple register allocation of only infinite-weight
// Variables while respecting pre-colored Variables. Some properties // Variables while respecting pre-colored Variables. Some properties we take
// we take advantage of: // advantage of:
// //
// * Live ranges of interest consist of a single segment. // * Live ranges of interest consist of a single segment.
// //
// * Live ranges of interest never span a call instruction. // * Live ranges of interest never span a call instruction.
// //
// * Phi instructions are not considered because either phis have // * Phi instructions are not considered because either phis have already been
// already been lowered, or they don't contain any pre-colored or // lowered, or they don't contain any pre-colored or infinite-weight
// infinite-weight Variables. // Variables.
// //
// * We don't need to renumber instructions before computing live // * We don't need to renumber instructions before computing live ranges
// ranges because all the high-level ICE instructions are deleted // because all the high-level ICE instructions are deleted prior to lowering,
// prior to lowering, and the low-level instructions are added in // and the low-level instructions are added in monotonically increasing order.
// monotonically increasing order.
// //
// * There are no opportunities for register preference or allowing // * There are no opportunities for register preference or allowing overlap.
// overlap.
// //
// Some properties we aren't (yet) taking advantage of: // Some properties we aren't (yet) taking advantage of:
// //
// * Because live ranges are a single segment, the Inactive set will // * Because live ranges are a single segment, the Inactive set will always be
// always be empty, and the live range trimming operation is // empty, and the live range trimming operation is unnecessary.
// unnecessary.
// //
// * Calculating overlap of single-segment live ranges could be // * Calculating overlap of single-segment live ranges could be optimized a
// optimized a bit. // bit.
void LinearScan::initForInfOnly() { void LinearScan::initForInfOnly() {
TimerMarker T(TimerStack::TT_initUnhandled, Func); TimerMarker T(TimerStack::TT_initUnhandled, Func);
FindPreference = false; FindPreference = false;
...@@ -168,9 +163,8 @@ void LinearScan::initForInfOnly() { ...@@ -168,9 +163,8 @@ void LinearScan::initForInfOnly() {
SizeT NumVars = 0; SizeT NumVars = 0;
const VarList &Vars = Func->getVariables(); const VarList &Vars = Func->getVariables();
// Iterate across all instructions and record the begin and end of // Iterate across all instructions and record the begin and end of the live
// the live range for each variable that is pre-colored or infinite // range for each variable that is pre-colored or infinite weight.
// weight.
std::vector<InstNumberT> LRBegin(Vars.size(), Inst::NumberSentinel); std::vector<InstNumberT> LRBegin(Vars.size(), Inst::NumberSentinel);
std::vector<InstNumberT> LREnd(Vars.size(), Inst::NumberSentinel); std::vector<InstNumberT> LREnd(Vars.size(), Inst::NumberSentinel);
for (CfgNode *Node : Func->getNodes()) { for (CfgNode *Node : Func->getNodes()) {
...@@ -219,12 +213,12 @@ void LinearScan::initForInfOnly() { ...@@ -219,12 +213,12 @@ void LinearScan::initForInfOnly() {
--NumVars; --NumVars;
} }
} }
// This isn't actually a fatal condition, but it would be nice to // This isn't actually a fatal condition, but it would be nice to know if we
// know if we somehow pre-calculated Unhandled's size wrong. // somehow pre-calculated Unhandled's size wrong.
assert(NumVars == 0); assert(NumVars == 0);
// Don't build up the list of Kills because we know that no // Don't build up the list of Kills because we know that no infinite-weight
// infinite-weight Variable has a live range spanning a call. // Variable has a live range spanning a call.
Kills.clear(); Kills.clear();
} }
...@@ -271,25 +265,25 @@ void LinearScan::init(RegAllocKind Kind) { ...@@ -271,25 +265,25 @@ void LinearScan::init(RegAllocKind Kind) {
// is not explicitly used during Cur's live range, spill that register to a // is not explicitly used during Cur's live range, spill that register to a
// stack location right before Cur's live range begins, and fill (reload) the // stack location right before Cur's live range begins, and fill (reload) the
// register from the stack location right after Cur's live range ends. // register from the stack location right after Cur's live range ends.
void LinearScan::addSpillFill(Variable *Cur, llvm::SmallBitVector RegMask) { void LinearScan::addSpillFill(IterationState &Iter) {
// Identify the actual instructions that begin and end Cur's live range. // Identify the actual instructions that begin and end Iter.Cur's live range.
// Iterate through Cur's node's instruction list until we find the actual // Iterate through Iter.Cur's node's instruction list until we find the actual
// instructions with instruction numbers corresponding to Cur's recorded live // instructions with instruction numbers corresponding to Iter.Cur's recorded
// range endpoints. This sounds inefficient but shouldn't be a problem in // live range endpoints. This sounds inefficient but shouldn't be a problem
// practice because: // in practice because:
// (1) This function is almost never called in practice. // (1) This function is almost never called in practice.
// (2) Since this register over-subscription problem happens only for // (2) Since this register over-subscription problem happens only for
// phi-lowered instructions, the number of instructions in the node is // phi-lowered instructions, the number of instructions in the node is
// proportional to the number of phi instructions in the original node, // proportional to the number of phi instructions in the original node,
// which is never very large in practice. // which is never very large in practice.
// (3) We still have to iterate through all instructions of Cur's live range // (3) We still have to iterate through all instructions of Iter.Cur's live
// to find all explicitly used registers (though the live range is usually // range to find all explicitly used registers (though the live range is
// only 2-3 instructions), so the main cost that could be avoided would be // usually only 2-3 instructions), so the main cost that could be avoided
// finding the instruction that begin's Cur's live range. // would be finding the instruction that begin's Iter.Cur's live range.
assert(!Cur->getLiveRange().isEmpty()); assert(!Iter.Cur->getLiveRange().isEmpty());
InstNumberT Start = Cur->getLiveRange().getStart(); InstNumberT Start = Iter.Cur->getLiveRange().getStart();
InstNumberT End = Cur->getLiveRange().getEnd(); InstNumberT End = Iter.Cur->getLiveRange().getEnd();
CfgNode *Node = Func->getVMetadata()->getLocalUseNode(Cur); CfgNode *Node = Func->getVMetadata()->getLocalUseNode(Iter.Cur);
assert(Node); assert(Node);
InstList &Insts = Node->getInsts(); InstList &Insts = Node->getInsts();
InstList::iterator SpillPoint = Insts.end(); InstList::iterator SpillPoint = Insts.end();
...@@ -311,7 +305,7 @@ void LinearScan::addSpillFill(Variable *Cur, llvm::SmallBitVector RegMask) { ...@@ -311,7 +305,7 @@ void LinearScan::addSpillFill(Variable *Cur, llvm::SmallBitVector RegMask) {
for (SizeT j = 0; j < NumVars; ++j) { for (SizeT j = 0; j < NumVars; ++j) {
const Variable *Var = Src->getVar(j); const Variable *Var = Src->getVar(j);
if (Var->hasRegTmp()) if (Var->hasRegTmp())
RegMask[Var->getRegNumTmp()] = false; Iter.RegMask[Var->getRegNumTmp()] = false;
} }
} }
} }
...@@ -320,14 +314,14 @@ void LinearScan::addSpillFill(Variable *Cur, llvm::SmallBitVector RegMask) { ...@@ -320,14 +314,14 @@ void LinearScan::addSpillFill(Variable *Cur, llvm::SmallBitVector RegMask) {
assert(FillPoint != Insts.end()); assert(FillPoint != Insts.end());
++FillPoint; ++FillPoint;
// TODO(stichnot): Randomize instead of find_first(). // TODO(stichnot): Randomize instead of find_first().
int32_t RegNum = RegMask.find_first(); int32_t RegNum = Iter.RegMask.find_first();
assert(RegNum != -1); assert(RegNum != -1);
Cur->setRegNumTmp(RegNum); Iter.Cur->setRegNumTmp(RegNum);
TargetLowering *Target = Func->getTarget(); TargetLowering *Target = Func->getTarget();
Variable *Preg = Target->getPhysicalRegister(RegNum, Cur->getType()); Variable *Preg = Target->getPhysicalRegister(RegNum, Iter.Cur->getType());
// TODO(stichnot): Add SpillLoc to VariablesMetadata tracking so that SpillLoc // TODO(stichnot): Add SpillLoc to VariablesMetadata tracking so that SpillLoc
// is correctly identified as !isMultiBlock(), reducing stack frame size. // is correctly identified as !isMultiBlock(), reducing stack frame size.
Variable *SpillLoc = Func->makeVariable(Cur->getType()); Variable *SpillLoc = Func->makeVariable(Iter.Cur->getType());
// Add "reg=FakeDef;spill=reg" before SpillPoint // Add "reg=FakeDef;spill=reg" before SpillPoint
Target->lowerInst(Node, SpillPoint, InstFakeDef::create(Func, Preg)); Target->lowerInst(Node, SpillPoint, InstFakeDef::create(Func, Preg));
Target->lowerInst(Node, SpillPoint, InstAssign::create(Func, SpillLoc, Preg)); Target->lowerInst(Node, SpillPoint, InstAssign::create(Func, SpillLoc, Preg));
...@@ -336,92 +330,7 @@ void LinearScan::addSpillFill(Variable *Cur, llvm::SmallBitVector RegMask) { ...@@ -336,92 +330,7 @@ void LinearScan::addSpillFill(Variable *Cur, llvm::SmallBitVector RegMask) {
Target->lowerInst(Node, FillPoint, InstFakeUse::create(Func, Preg)); Target->lowerInst(Node, FillPoint, InstFakeUse::create(Func, Preg));
} }
// Implements the linear-scan algorithm. Based on "Linear Scan void LinearScan::handleActiveRangeExpiredOrInactive(const Variable *Cur) {
// Register Allocation in the Context of SSA Form and Register
// Constraints" by Hanspeter Mössenböck and Michael Pfeiffer,
// ftp://ftp.ssw.uni-linz.ac.at/pub/Papers/Moe02.PDF . This
// implementation is modified to take affinity into account and allow
// two interfering variables to share the same register in certain
// cases.
//
// Requires running Cfg::liveness(Liveness_Intervals) in
// preparation. Results are assigned to Variable::RegNum for each
// Variable.
void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull,
bool Randomized) {
TimerMarker T(TimerStack::TT_linearScan, Func);
assert(RegMaskFull.any()); // Sanity check
GlobalContext *Ctx = Func->getContext();
const bool Verbose = BuildDefs::dump() && Func->isVerbose(IceV_LinearScan);
if (Verbose)
Ctx->lockStr();
Func->resetCurrentNode();
VariablesMetadata *VMetadata = Func->getVMetadata();
const size_t NumRegisters = RegMaskFull.size();
llvm::SmallBitVector PreDefinedRegisters(NumRegisters);
if (Randomized) {
for (Variable *Var : UnhandledPrecolored) {
PreDefinedRegisters[Var->getRegNum()] = true;
}
}
// Build a LiveRange representing the Kills list.
LiveRange KillsRange(Kills);
KillsRange.untrim();
// RegUses[I] is the number of live ranges (variables) that register
// I is currently assigned to. It can be greater than 1 as a result
// of AllowOverlap inference below.
llvm::SmallVector<int, REGS_SIZE> RegUses(NumRegisters);
// Unhandled is already set to all ranges in increasing order of
// start points.
assert(Active.empty());
assert(Inactive.empty());
assert(Handled.empty());
const TargetLowering::RegSetMask RegsInclude =
TargetLowering::RegSet_CallerSave;
const TargetLowering::RegSetMask RegsExclude = TargetLowering::RegSet_None;
const llvm::SmallBitVector KillsMask =
Func->getTarget()->getRegisterSet(RegsInclude, RegsExclude);
while (!Unhandled.empty()) {
Variable *Cur = Unhandled.back();
Unhandled.pop_back();
if (Verbose) {
Ostream &Str = Ctx->getStrDump();
Str << "\nConsidering ";
dumpLiveRange(Cur, Func);
Str << "\n";
}
const llvm::SmallBitVector RegMask =
RegMaskFull & Func->getTarget()->getRegisterSetForType(Cur->getType());
KillsRange.trim(Cur->getLiveRange().getStart());
// Check for pre-colored ranges. If Cur is pre-colored, it
// definitely gets that register. Previously processed live
// ranges would have avoided that register due to it being
// pre-colored. Future processed live ranges won't evict that
// register because the live range has infinite weight.
if (Cur->hasReg()) {
int32_t RegNum = Cur->getRegNum();
// RegNumTmp should have already been set above.
assert(Cur->getRegNumTmp() == RegNum);
if (Verbose) {
Ostream &Str = Ctx->getStrDump();
Str << "Precoloring ";
dumpLiveRange(Cur, Func);
Str << "\n";
}
Active.push_back(Cur);
assert(RegUses[RegNum] >= 0);
++RegUses[RegNum];
assert(!UnhandledPrecolored.empty());
assert(UnhandledPrecolored.back() == Cur);
UnhandledPrecolored.pop_back();
continue;
}
// Check for active ranges that have expired or become inactive.
for (SizeT I = Active.size(); I > 0; --I) { for (SizeT I = Active.size(); I > 0; --I) {
const SizeT Index = I - 1; const SizeT Index = I - 1;
Variable *Item = Active[Index]; Variable *Item = Active[Index];
...@@ -429,22 +338,12 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull, ...@@ -429,22 +338,12 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull,
bool Moved = false; bool Moved = false;
if (Item->rangeEndsBefore(Cur)) { if (Item->rangeEndsBefore(Cur)) {
// Move Item from Active to Handled list. // Move Item from Active to Handled list.
if (Verbose) { dumpLiveRangeTrace("Expiring ", Cur);
Ostream &Str = Ctx->getStrDump();
Str << "Expiring ";
dumpLiveRange(Item, Func);
Str << "\n";
}
moveItem(Active, Index, Handled); moveItem(Active, Index, Handled);
Moved = true; Moved = true;
} else if (!Item->rangeOverlapsStart(Cur)) { } else if (!Item->rangeOverlapsStart(Cur)) {
// Move Item from Active to Inactive list. // Move Item from Active to Inactive list.
if (Verbose) { dumpLiveRangeTrace("Inactivating ", Cur);
Ostream &Str = Ctx->getStrDump();
Str << "Inactivating ";
dumpLiveRange(Item, Func);
Str << "\n";
}
moveItem(Active, Index, Inactive); moveItem(Active, Index, Inactive);
Moved = true; Moved = true;
} }
...@@ -456,29 +355,20 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull, ...@@ -456,29 +355,20 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull,
assert(RegUses[RegNum] >= 0); assert(RegUses[RegNum] >= 0);
} }
} }
}
// Check for inactive ranges that have expired or reactivated. void LinearScan::handleInactiveRangeExpiredOrReactivated(const Variable *Cur) {
for (SizeT I = Inactive.size(); I > 0; --I) { for (SizeT I = Inactive.size(); I > 0; --I) {
const SizeT Index = I - 1; const SizeT Index = I - 1;
Variable *Item = Inactive[Index]; Variable *Item = Inactive[Index];
Item->trimLiveRange(Cur->getLiveRange().getStart()); Item->trimLiveRange(Cur->getLiveRange().getStart());
if (Item->rangeEndsBefore(Cur)) { if (Item->rangeEndsBefore(Cur)) {
// Move Item from Inactive to Handled list. // Move Item from Inactive to Handled list.
if (Verbose) { dumpLiveRangeTrace("Expiring ", Cur);
Ostream &Str = Ctx->getStrDump();
Str << "Expiring ";
dumpLiveRange(Item, Func);
Str << "\n";
}
moveItem(Inactive, Index, Handled); moveItem(Inactive, Index, Handled);
} else if (Item->rangeOverlapsStart(Cur)) { } else if (Item->rangeOverlapsStart(Cur)) {
// Move Item from Inactive to Active list. // Move Item from Inactive to Active list.
if (Verbose) { dumpLiveRangeTrace("Reactivating ", Cur);
Ostream &Str = Ctx->getStrDump();
Str << "Reactivating ";
dumpLiveRange(Item, Func);
Str << "\n";
}
moveItem(Inactive, Index, Active); moveItem(Inactive, Index, Active);
// Increment Item in RegUses[]. // Increment Item in RegUses[].
assert(Item->hasRegTmp()); assert(Item->hasRegTmp());
...@@ -487,238 +377,187 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull, ...@@ -487,238 +377,187 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull,
++RegUses[RegNum]; ++RegUses[RegNum];
} }
} }
}
// Infer register preference and allowable overlap. Only form a preference when
// the current Variable has an unambiguous "first" definition. The preference
// is some source Variable of the defining instruction that either is assigned
// a register that is currently free, or that is assigned a register that is
// not free but overlap is allowed. Overlap is allowed when the Variable under
// consideration is single-definition, and its definition is a simple
// assignment - i.e., the register gets copied/aliased but is never modified.
// Furthermore, overlap is only allowed when preferred Variable definition
// instructions do not appear within the current Variable's live range.
void LinearScan::findRegisterPreference(IterationState &Iter) {
Iter.Prefer = nullptr;
Iter.PreferReg = Variable::NoRegister;
Iter.AllowOverlap = false;
// Calculate available registers into Free[].
llvm::SmallBitVector Free = RegMask;
for (SizeT i = 0; i < RegMask.size(); ++i) {
if (RegUses[i] > 0)
Free[i] = false;
}
// Infer register preference and allowable overlap. Only form a
// preference when the current Variable has an unambiguous "first"
// definition. The preference is some source Variable of the
// defining instruction that either is assigned a register that is
// currently free, or that is assigned a register that is not free
// but overlap is allowed. Overlap is allowed when the Variable
// under consideration is single-definition, and its definition is
// a simple assignment - i.e., the register gets copied/aliased
// but is never modified. Furthermore, overlap is only allowed
// when preferred Variable definition instructions do not appear
// within the current Variable's live range.
Variable *Prefer = nullptr;
int32_t PreferReg = Variable::NoRegister;
bool AllowOverlap = false;
if (FindPreference) { if (FindPreference) {
if (const Inst *DefInst = VMetadata->getFirstDefinition(Cur)) { VariablesMetadata *VMetadata = Func->getVMetadata();
assert(DefInst->getDest() == Cur); if (const Inst *DefInst = VMetadata->getFirstDefinition(Iter.Cur)) {
assert(DefInst->getDest() == Iter.Cur);
bool IsAssign = DefInst->isSimpleAssign(); bool IsAssign = DefInst->isSimpleAssign();
bool IsSingleDef = !VMetadata->isMultiDef(Cur); bool IsSingleDef = !VMetadata->isMultiDef(Iter.Cur);
for (SizeT i = 0; i < DefInst->getSrcSize(); ++i) { for (SizeT i = 0; i < DefInst->getSrcSize(); ++i) {
// TODO(stichnot): Iterate through the actual Variables of the // TODO(stichnot): Iterate through the actual Variables of the
// instruction, not just the source operands. This could // instruction, not just the source operands. This could capture Load
// capture Load instructions, including address mode // instructions, including address mode optimization, for Prefer (but
// optimization, for Prefer (but not for AllowOverlap). // not for AllowOverlap).
if (Variable *SrcVar = llvm::dyn_cast<Variable>(DefInst->getSrc(i))) { if (Variable *SrcVar = llvm::dyn_cast<Variable>(DefInst->getSrc(i))) {
int32_t SrcReg = SrcVar->getRegNumTmp(); int32_t SrcReg = SrcVar->getRegNumTmp();
// Only consider source variables that have (so far) been // Only consider source variables that have (so far) been assigned a
// assigned a register. That register must be one in the // register. That register must be one in the RegMask set, e.g.
// RegMask set, e.g. don't try to prefer the stack pointer // don't try to prefer the stack pointer as a result of the stacksave
// as a result of the stacksave intrinsic. // intrinsic.
if (SrcVar->hasRegTmp() && RegMask[SrcReg]) { if (SrcVar->hasRegTmp() && Iter.RegMask[SrcReg]) {
if (FindOverlap && !Free[SrcReg]) { if (FindOverlap && !Iter.Free[SrcReg]) {
// Don't bother trying to enable AllowOverlap if the // Don't bother trying to enable AllowOverlap if the register is
// register is already free. // already free.
AllowOverlap = Iter.AllowOverlap = IsSingleDef && IsAssign &&
IsSingleDef && IsAssign && !overlapsDefs(Func, Cur, SrcVar); !overlapsDefs(Func, Iter.Cur, SrcVar);
} }
if (AllowOverlap || Free[SrcReg]) { if (Iter.AllowOverlap || Iter.Free[SrcReg]) {
Prefer = SrcVar; Iter.Prefer = SrcVar;
PreferReg = SrcReg; Iter.PreferReg = SrcReg;
} }
} }
} }
} }
if (Verbose && Prefer) { if (Verbose && Iter.Prefer) {
Ostream &Str = Ctx->getStrDump(); Ostream &Str = Ctx->getStrDump();
Str << "Initial Prefer="; Str << "Initial Iter.Prefer=";
Prefer->dump(Func); Iter.Prefer->dump(Func);
Str << " R=" << PreferReg << " LIVE=" << Prefer->getLiveRange() Str << " R=" << Iter.PreferReg
<< " Overlap=" << AllowOverlap << "\n"; << " LIVE=" << Iter.Prefer->getLiveRange()
<< " Overlap=" << Iter.AllowOverlap << "\n";
} }
} }
} }
}
// Remove registers from the Free[] list where an Inactive range // Remove registers from the Free[] list where an Inactive range overlaps with
// overlaps with the current range. // the current range.
void LinearScan::filterFreeWithInactiveRanges(IterationState &Iter) {
for (const Variable *Item : Inactive) { for (const Variable *Item : Inactive) {
if (Item->rangeOverlaps(Cur)) { if (Item->rangeOverlaps(Iter.Cur)) {
int32_t RegNum = Item->getRegNumTmp(); int32_t RegNum = Item->getRegNumTmp();
// Don't assert(Free[RegNum]) because in theory (though // Don't assert(Free[RegNum]) because in theory (though probably never in
// probably never in practice) there could be two inactive // practice) there could be two inactive variables that were marked with
// variables that were marked with AllowOverlap. // AllowOverlap.
Free[RegNum] = false; Iter.Free[RegNum] = false;
// Disable AllowOverlap if an Inactive variable, which is not // Disable AllowOverlap if an Inactive variable, which is not Prefer,
// Prefer, shares Prefer's register, and has a definition // shares Prefer's register, and has a definition within Cur's live
// within Cur's live range. // range.
if (AllowOverlap && Item != Prefer && RegNum == PreferReg && if (Iter.AllowOverlap && Item != Iter.Prefer &&
overlapsDefs(Func, Cur, Item)) { RegNum == Iter.PreferReg && overlapsDefs(Func, Iter.Cur, Item)) {
AllowOverlap = false; Iter.AllowOverlap = false;
dumpDisableOverlap(Func, Item, "Inactive"); dumpDisableOverlap(Func, Item, "Inactive");
} }
} }
} }
}
// Disable AllowOverlap if an Active variable, which is not // Remove registers from the Free[] list where an Unhandled pre-colored range
// Prefer, shares Prefer's register, and has a definition within // overlaps with the current range, and set those registers to infinite weight
// Cur's live range. // so that they aren't candidates for eviction. Cur->rangeEndsBefore(Item) is
if (AllowOverlap) { // an early exit check that turns a guaranteed O(N^2) algorithm into expected
for (const Variable *Item : Active) { // linear complexity.
int32_t RegNum = Item->getRegNumTmp(); void LinearScan::filterFreeWithPrecoloredRanges(IterationState &Iter) {
if (Item != Prefer && RegNum == PreferReg &&
overlapsDefs(Func, Cur, Item)) {
AllowOverlap = false;
dumpDisableOverlap(Func, Item, "Active");
}
}
}
llvm::SmallVector<RegWeight, REGS_SIZE> Weights(RegMask.size());
// Remove registers from the Free[] list where an Unhandled
// pre-colored range overlaps with the current range, and set those
// registers to infinite weight so that they aren't candidates for
// eviction. Cur->rangeEndsBefore(Item) is an early exit check
// that turns a guaranteed O(N^2) algorithm into expected linear
// complexity.
llvm::SmallBitVector PrecoloredUnhandledMask(RegMask.size());
// Note: PrecoloredUnhandledMask is only used for dumping.
for (Variable *Item : reverse_range(UnhandledPrecolored)) { for (Variable *Item : reverse_range(UnhandledPrecolored)) {
assert(Item->hasReg()); assert(Item->hasReg());
if (Cur->rangeEndsBefore(Item)) if (Iter.Cur->rangeEndsBefore(Item))
break; break;
if (Item->rangeOverlaps(Cur)) { if (Item->rangeOverlaps(Iter.Cur)) {
int32_t ItemReg = Item->getRegNum(); // Note: not getRegNumTmp() int32_t ItemReg = Item->getRegNum(); // Note: not getRegNumTmp()
Weights[ItemReg].setWeight(RegWeight::Inf); Iter.Weights[ItemReg].setWeight(RegWeight::Inf);
Free[ItemReg] = false; Iter.Free[ItemReg] = false;
PrecoloredUnhandledMask[ItemReg] = true; Iter.PrecoloredUnhandledMask[ItemReg] = true;
// Disable AllowOverlap if the preferred register is one of // Disable Iter.AllowOverlap if the preferred register is one of these
// these pre-colored unhandled overlapping ranges. // pre-colored unhandled overlapping ranges.
if (AllowOverlap && ItemReg == PreferReg) { if (Iter.AllowOverlap && ItemReg == Iter.PreferReg) {
AllowOverlap = false; Iter.AllowOverlap = false;
dumpDisableOverlap(Func, Item, "PrecoloredUnhandled"); dumpDisableOverlap(Func, Item, "PrecoloredUnhandled");
} }
} }
} }
}
// Remove scratch registers from the Free[] list, and mark their void LinearScan::allocatePrecoloredRegister(Variable *Cur) {
// Weights[] as infinite, if KillsRange overlaps Cur's live range. int32_t RegNum = Cur->getRegNum();
constexpr bool UseTrimmed = true; // RegNumTmp should have already been set above.
if (Cur->getLiveRange().overlaps(KillsRange, UseTrimmed)) { assert(Cur->getRegNumTmp() == RegNum);
Free.reset(KillsMask); dumpLiveRangeTrace("Precoloring ", Cur);
for (int i = KillsMask.find_first(); i != -1; Active.push_back(Cur);
i = KillsMask.find_next(i)) { assert(RegUses[RegNum] >= 0);
Weights[i].setWeight(RegWeight::Inf); ++RegUses[RegNum];
if (PreferReg == i) assert(!UnhandledPrecolored.empty());
AllowOverlap = false; assert(UnhandledPrecolored.back() == Cur);
} UnhandledPrecolored.pop_back();
} }
// Print info about physical register availability. void LinearScan::allocatePreferredRegister(IterationState &Iter) {
if (Verbose) { Iter.Cur->setRegNumTmp(Iter.PreferReg);
Ostream &Str = Ctx->getStrDump(); dumpLiveRangeTrace("Preferring ", Iter.Cur);
for (SizeT i = 0; i < RegMask.size(); ++i) { assert(RegUses[Iter.PreferReg] >= 0);
if (RegMask[i]) { ++RegUses[Iter.PreferReg];
Str << Func->getTarget()->getRegName(i, IceType_i32) Active.push_back(Iter.Cur);
<< "(U=" << RegUses[i] << ",F=" << Free[i] }
<< ",P=" << PrecoloredUnhandledMask[i] << ") ";
}
}
Str << "\n";
}
if (Prefer && (AllowOverlap || Free[PreferReg])) { void LinearScan::allocateFreeRegister(IterationState &Iter) {
// First choice: a preferred register that is either free or is int32_t RegNum = Iter.Free.find_first();
// allowed to overlap with its linked variable. Iter.Cur->setRegNumTmp(RegNum);
Cur->setRegNumTmp(PreferReg); dumpLiveRangeTrace("Allocating ", Iter.Cur);
if (Verbose) {
Ostream &Str = Ctx->getStrDump();
Str << "Preferring ";
dumpLiveRange(Cur, Func);
Str << "\n";
}
assert(RegUses[PreferReg] >= 0);
++RegUses[PreferReg];
Active.push_back(Cur);
} else if (Free.any()) {
// Second choice: any free register. TODO: After explicit
// affinity is considered, is there a strategy better than just
// picking the lowest-numbered available register?
int32_t RegNum = Free.find_first();
Cur->setRegNumTmp(RegNum);
if (Verbose) {
Ostream &Str = Ctx->getStrDump();
Str << "Allocating ";
dumpLiveRange(Cur, Func);
Str << "\n";
}
assert(RegUses[RegNum] >= 0); assert(RegUses[RegNum] >= 0);
++RegUses[RegNum]; ++RegUses[RegNum];
Active.push_back(Cur); Active.push_back(Iter.Cur);
} else { }
// Fallback: there are no free registers, so we look for the
// lowest-weight register and see if Cur has higher weight. void LinearScan::handleNoFreeRegisters(IterationState &Iter) {
// Check Active ranges. // Check Active ranges.
for (const Variable *Item : Active) { for (const Variable *Item : Active) {
assert(Item->rangeOverlaps(Cur)); assert(Item->rangeOverlaps(Iter.Cur));
int32_t RegNum = Item->getRegNumTmp(); int32_t RegNum = Item->getRegNumTmp();
assert(Item->hasRegTmp()); assert(Item->hasRegTmp());
Weights[RegNum].addWeight(Item->getLiveRange().getWeight()); Iter.Weights[RegNum].addWeight(Item->getLiveRange().getWeight());
} }
// Same as above, but check Inactive ranges instead of Active. // Same as above, but check Inactive ranges instead of Active.
for (const Variable *Item : Inactive) { for (const Variable *Item : Inactive) {
int32_t RegNum = Item->getRegNumTmp(); int32_t RegNum = Item->getRegNumTmp();
assert(Item->hasRegTmp()); assert(Item->hasRegTmp());
if (Item->rangeOverlaps(Cur)) if (Item->rangeOverlaps(Iter.Cur))
Weights[RegNum].addWeight(Item->getLiveRange().getWeight()); Iter.Weights[RegNum].addWeight(Item->getLiveRange().getWeight());
} }
// All the weights are now calculated. Find the register with // All the weights are now calculated. Find the register with smallest
// smallest weight. // weight.
int32_t MinWeightIndex = RegMask.find_first(); int32_t MinWeightIndex = Iter.RegMask.find_first();
// MinWeightIndex must be valid because of the initial // MinWeightIndex must be valid because of the initial RegMask.any() test.
// RegMask.any() test.
assert(MinWeightIndex >= 0); assert(MinWeightIndex >= 0);
for (SizeT i = MinWeightIndex + 1; i < Weights.size(); ++i) { for (SizeT i = MinWeightIndex + 1; i < Iter.Weights.size(); ++i) {
if (RegMask[i] && Weights[i] < Weights[MinWeightIndex]) if (Iter.RegMask[i] && Iter.Weights[i] < Iter.Weights[MinWeightIndex])
MinWeightIndex = i; MinWeightIndex = i;
} }
if (Cur->getLiveRange().getWeight() <= Weights[MinWeightIndex]) { if (Iter.Cur->getLiveRange().getWeight() <= Iter.Weights[MinWeightIndex]) {
// Cur doesn't have priority over any other live ranges, so // Cur doesn't have priority over any other live ranges, so don't allocate
// don't allocate any register to it, and move it to the // any register to it, and move it to the Handled state.
// Handled state. Handled.push_back(Iter.Cur);
Handled.push_back(Cur); if (Iter.Cur->getLiveRange().getWeight().isInf()) {
if (Cur->getLiveRange().getWeight().isInf()) {
if (Kind == RAK_Phi) if (Kind == RAK_Phi)
addSpillFill(Cur, RegMask); addSpillFill(Iter);
else else
Func->setError("Unable to find a physical register for an " Func->setError("Unable to find a physical register for an "
"infinite-weight live range"); "infinite-weight live range");
} }
} else { } else {
// Evict all live ranges in Active that register number // Evict all live ranges in Active that register number MinWeightIndex is
// MinWeightIndex is assigned to. // assigned to.
for (SizeT I = Active.size(); I > 0; --I) { for (SizeT I = Active.size(); I > 0; --I) {
const SizeT Index = I - 1; const SizeT Index = I - 1;
Variable *Item = Active[Index]; Variable *Item = Active[Index];
if (Item->getRegNumTmp() == MinWeightIndex) { if (Item->getRegNumTmp() == MinWeightIndex) {
if (Verbose) { dumpLiveRangeTrace("Evicting ", Item);
Ostream &Str = Ctx->getStrDump();
Str << "Evicting ";
dumpLiveRange(Item, Func);
Str << "\n";
}
--RegUses[MinWeightIndex]; --RegUses[MinWeightIndex];
assert(RegUses[MinWeightIndex] >= 0); assert(RegUses[MinWeightIndex] >= 0);
Item->setRegNumTmp(Variable::NoRegister); Item->setRegNumTmp(Variable::NoRegister);
...@@ -730,65 +569,47 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull, ...@@ -730,65 +569,47 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull,
const SizeT Index = I - 1; const SizeT Index = I - 1;
Variable *Item = Inactive[Index]; Variable *Item = Inactive[Index];
// Note: The Item->rangeOverlaps(Cur) clause is not part of the // Note: The Item->rangeOverlaps(Cur) clause is not part of the
// description of AssignMemLoc() in the original paper. But // description of AssignMemLoc() in the original paper. But there
// there doesn't seem to be any need to evict an inactive // doesn't seem to be any need to evict an inactive live range that
// live range that doesn't overlap with the live range // doesn't overlap with the live range currently being considered. It's
// currently being considered. It's especially bad if we // especially bad if we would end up evicting an infinite-weight but
// would end up evicting an infinite-weight but // currently-inactive live range. The most common situation for this
// currently-inactive live range. The most common situation // would be a scratch register kill set for call instructions.
// for this would be a scratch register kill set for call
// instructions.
if (Item->getRegNumTmp() == MinWeightIndex && if (Item->getRegNumTmp() == MinWeightIndex &&
Item->rangeOverlaps(Cur)) { Item->rangeOverlaps(Iter.Cur)) {
if (Verbose) { dumpLiveRangeTrace("Evicting ", Item);
Ostream &Str = Ctx->getStrDump();
Str << "Evicting ";
dumpLiveRange(Item, Func);
Str << "\n";
}
Item->setRegNumTmp(Variable::NoRegister); Item->setRegNumTmp(Variable::NoRegister);
moveItem(Inactive, Index, Handled); moveItem(Inactive, Index, Handled);
} }
} }
// Assign the register to Cur. // Assign the register to Cur.
Cur->setRegNumTmp(MinWeightIndex); Iter.Cur->setRegNumTmp(MinWeightIndex);
assert(RegUses[MinWeightIndex] >= 0); assert(RegUses[MinWeightIndex] >= 0);
++RegUses[MinWeightIndex]; ++RegUses[MinWeightIndex];
Active.push_back(Cur); Active.push_back(Iter.Cur);
if (Verbose) { dumpLiveRangeTrace("Allocating ", Iter.Cur);
Ostream &Str = Ctx->getStrDump();
Str << "Allocating ";
dumpLiveRange(Cur, Func);
Str << "\n";
}
} }
} }
dump(Func);
}
// Move anything Active or Inactive to Handled for easier handling.
for (Variable *I : Active)
Handled.push_back(I);
Active.clear();
for (Variable *I : Inactive)
Handled.push_back(I);
Inactive.clear();
dump(Func);
void LinearScan::assignFinalRegisters(
const llvm::SmallBitVector &RegMaskFull,
const llvm::SmallBitVector &PreDefinedRegisters, bool Randomized) {
const size_t NumRegisters = RegMaskFull.size();
llvm::SmallVector<int32_t, REGS_SIZE> Permutation(NumRegisters); llvm::SmallVector<int32_t, REGS_SIZE> Permutation(NumRegisters);
if (Randomized) { if (Randomized) {
// Create a random number generator for regalloc randomization. Merge // Create a random number generator for regalloc randomization. Merge
// function's sequence and Kind value as the Salt. Because regAlloc() // function's sequence and Kind value as the Salt. Because regAlloc() is
// is called twice under O2, the second time with RAK_Phi, we check // called twice under O2, the second time with RAK_Phi, we check
// Kind == RAK_Phi to determine the lowest-order bit to make sure the // Kind == RAK_Phi to determine the lowest-order bit to make sure the Salt
// Salt is different. // is different.
uint64_t Salt = uint64_t Salt =
(Func->getSequenceNumber() << 1) ^ (Kind == RAK_Phi ? 0u : 1u); (Func->getSequenceNumber() << 1) ^ (Kind == RAK_Phi ? 0u : 1u);
Func->getTarget()->makeRandomRegisterPermutation( Func->getTarget()->makeRandomRegisterPermutation(
Permutation, PreDefinedRegisters | ~RegMaskFull, Salt); Permutation, PreDefinedRegisters | ~RegMaskFull, Salt);
} }
// Finish up by assigning RegNumTmp->RegNum (or a random permutation // Finish up by setting RegNum = RegNumTmp (or a random permutation thereof)
// thereof) for each Variable. // for each Variable.
for (Variable *Item : Handled) { for (Variable *Item : Handled) {
int32_t RegNum = Item->getRegNumTmp(); int32_t RegNum = Item->getRegNumTmp();
int32_t AssignedRegNum = RegNum; int32_t AssignedRegNum = RegNum;
...@@ -813,17 +634,167 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull, ...@@ -813,17 +634,167 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull,
} }
Item->setRegNum(AssignedRegNum); Item->setRegNum(AssignedRegNum);
} }
}
// Implements the linear-scan algorithm. Based on "Linear Scan Register
// Allocation in the Context of SSA Form and Register Constraints" by Hanspeter
// Mössenböck and Michael Pfeiffer,
// ftp://ftp.ssw.uni-linz.ac.at/pub/Papers/Moe02.PDF. This implementation is
// modified to take affinity into account and allow two interfering variables
// to share the same register in certain cases.
//
// Requires running Cfg::liveness(Liveness_Intervals) in preparation. Results
// are assigned to Variable::RegNum for each Variable.
void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull,
bool Randomized) {
TimerMarker T(TimerStack::TT_linearScan, Func);
assert(RegMaskFull.any()); // Sanity check
if (Verbose)
Ctx->lockStr();
Func->resetCurrentNode();
const size_t NumRegisters = RegMaskFull.size();
llvm::SmallBitVector PreDefinedRegisters(NumRegisters);
if (Randomized) {
for (Variable *Var : UnhandledPrecolored) {
PreDefinedRegisters[Var->getRegNum()] = true;
}
}
// Build a LiveRange representing the Kills list.
LiveRange KillsRange(Kills);
KillsRange.untrim();
// Reset the register use count
RegUses.resize(NumRegisters);
std::fill(RegUses.begin(), RegUses.end(), 0);
// Unhandled is already set to all ranges in increasing order of start
// points.
assert(Active.empty());
assert(Inactive.empty());
assert(Handled.empty());
const TargetLowering::RegSetMask RegsInclude =
TargetLowering::RegSet_CallerSave;
const TargetLowering::RegSetMask RegsExclude = TargetLowering::RegSet_None;
const llvm::SmallBitVector KillsMask =
Func->getTarget()->getRegisterSet(RegsInclude, RegsExclude);
// Allocate memory once outside the loop
IterationState Iter;
Iter.Weights.reserve(NumRegisters);
Iter.PrecoloredUnhandledMask.reserve(NumRegisters);
while (!Unhandled.empty()) {
Iter.Cur = Unhandled.back();
Unhandled.pop_back();
dumpLiveRangeTrace("\nConsidering ", Iter.Cur);
Iter.RegMask =
RegMaskFull &
Func->getTarget()->getRegisterSetForType(Iter.Cur->getType());
KillsRange.trim(Iter.Cur->getLiveRange().getStart());
// Check for pre-colored ranges. If Cur is pre-colored, it definitely gets
// that register. Previously processed live ranges would have avoided that
// register due to it being pre-colored. Future processed live ranges won't
// evict that register because the live range has infinite weight.
if (Iter.Cur->hasReg()) {
allocatePrecoloredRegister(Iter.Cur);
continue;
}
handleActiveRangeExpiredOrInactive(Iter.Cur);
handleInactiveRangeExpiredOrReactivated(Iter.Cur);
// Calculate available registers into Free[].
Iter.Free = Iter.RegMask;
for (SizeT i = 0; i < Iter.RegMask.size(); ++i) {
if (RegUses[i] > 0)
Iter.Free[i] = false;
}
findRegisterPreference(Iter);
filterFreeWithInactiveRanges(Iter);
// Disable AllowOverlap if an Active variable, which is not Prefer, shares
// Prefer's register, and has a definition within Cur's live range.
if (Iter.AllowOverlap) {
for (const Variable *Item : Active) {
int32_t RegNum = Item->getRegNumTmp();
if (Item != Iter.Prefer && RegNum == Iter.PreferReg &&
overlapsDefs(Func, Iter.Cur, Item)) {
Iter.AllowOverlap = false;
dumpDisableOverlap(Func, Item, "Active");
}
}
}
Iter.Weights.resize(Iter.RegMask.size());
std::fill(Iter.Weights.begin(), Iter.Weights.end(), RegWeight());
Iter.PrecoloredUnhandledMask.resize(Iter.RegMask.size());
Iter.PrecoloredUnhandledMask.reset();
filterFreeWithPrecoloredRanges(Iter);
// Remove scratch registers from the Free[] list, and mark their Weights[]
// as infinite, if KillsRange overlaps Cur's live range.
constexpr bool UseTrimmed = true;
if (Iter.Cur->getLiveRange().overlaps(KillsRange, UseTrimmed)) {
Iter.Free.reset(KillsMask);
for (int i = KillsMask.find_first(); i != -1;
i = KillsMask.find_next(i)) {
Iter.Weights[i].setWeight(RegWeight::Inf);
if (Iter.PreferReg == i)
Iter.AllowOverlap = false;
}
}
// Print info about physical register availability.
if (Verbose) {
Ostream &Str = Ctx->getStrDump();
for (SizeT i = 0; i < Iter.RegMask.size(); ++i) {
if (Iter.RegMask[i]) {
Str << Func->getTarget()->getRegName(i, IceType_i32)
<< "(U=" << RegUses[i] << ",F=" << Iter.Free[i]
<< ",P=" << Iter.PrecoloredUnhandledMask[i] << ") ";
}
}
Str << "\n";
}
// TODO: Consider running register allocation one more time, with if (Iter.Prefer && (Iter.AllowOverlap || Iter.Free[Iter.PreferReg])) {
// infinite registers, for two reasons. First, evicted live ranges // First choice: a preferred register that is either free or is allowed
// get a second chance for a register. Second, it allows coalescing // to overlap with its linked variable.
// of stack slots. If there is no time budget for the second allocatePreferredRegister(Iter);
// register allocation run, each unallocated variable just gets its } else if (Iter.Free.any()) {
// own slot. // Second choice: any free register.
allocateFreeRegister(Iter);
} else {
// Fallback: there are no free registers, so we look for the
// lowest-weight register and see if Cur has higher weight.
handleNoFreeRegisters(Iter);
}
dump(Func);
}
// Move anything Active or Inactive to Handled for easier handling.
Handled.insert(Handled.end(), Active.begin(), Active.end());
Active.clear();
Handled.insert(Handled.end(), Inactive.begin(), Inactive.end());
Inactive.clear();
dump(Func);
assignFinalRegisters(RegMaskFull, PreDefinedRegisters, Randomized);
// TODO: Consider running register allocation one more time, with infinite
// registers, for two reasons. First, evicted live ranges get a second chance
// for a register. Second, it allows coalescing of stack slots. If there is
// no time budget for the second register allocation run, each unallocated
// variable just gets its own slot.
// //
// Another idea for coalescing stack slots is to initialize the // Another idea for coalescing stack slots is to initialize the Unhandled
// Unhandled list with just the unallocated variables, saving time // list with just the unallocated variables, saving time but not offering
// but not offering second-chance opportunities. // second-chance opportunities.
if (Verbose) if (Verbose)
Ctx->unlockStr(); Ctx->unlockStr();
...@@ -831,6 +802,18 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull, ...@@ -831,6 +802,18 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull,
// ======================== Dump routines ======================== // // ======================== Dump routines ======================== //
void LinearScan::dumpLiveRangeTrace(const char *Label, const Variable *Item) {
if (!BuildDefs::dump())
return;
if (Verbose) {
Ostream &Str = Ctx->getStrDump();
Str << Label;
dumpLiveRange(Item, Func);
Str << "\n";
}
}
void LinearScan::dump(Cfg *Func) const { void LinearScan::dump(Cfg *Func) const {
if (!BuildDefs::dump()) if (!BuildDefs::dump())
return; return;
......
src/IceRegAlloc.h
...@@ -8,9 +8,9 @@ ...@@ -8,9 +8,9 @@
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
/// ///
/// \file /// \file
/// This file declares the LinearScan data structure used during /// This file declares the LinearScan data structure used during linear-scan
/// linear-scan register allocation, which holds the various work /// register allocation, which holds the various work queues for the linear-scan
/// queues for the linear-scan algorithm. /// algorithm.
/// ///
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#define SUBZERO_SRC_ICEREGALLOC_H #define SUBZERO_SRC_ICEREGALLOC_H
#include "IceDefs.h" #include "IceDefs.h"
#include "IceOperand.h"
#include "IceTypes.h" #include "IceTypes.h"
namespace Ice { namespace Ice {
...@@ -28,42 +29,89 @@ class LinearScan { ...@@ -28,42 +29,89 @@ class LinearScan {
LinearScan &operator=(const LinearScan &) = delete; LinearScan &operator=(const LinearScan &) = delete;
public: public:
explicit LinearScan(Cfg *Func) : Func(Func) {} explicit LinearScan(Cfg *Func);
void init(RegAllocKind Kind); void init(RegAllocKind Kind);
void scan(const llvm::SmallBitVector &RegMask, bool Randomized); void scan(const llvm::SmallBitVector &RegMask, bool Randomized);
void dump(Cfg *Func) const; void dump(Cfg *Func) const;
// TODO(stichnot): Statically choose the size based on the target being
// compiled.
static constexpr size_t REGS_SIZE = 32;
private: private:
typedef std::vector<Variable *> OrderedRanges; typedef std::vector<Variable *> OrderedRanges;
typedef std::vector<Variable *> UnorderedRanges; typedef std::vector<Variable *> UnorderedRanges;
class IterationState {
IterationState(const IterationState &) = delete;
IterationState operator=(const IterationState &) = delete;
public:
IterationState() = default;
Variable *Cur = nullptr;
Variable *Prefer = nullptr;
int32_t PreferReg = Variable::NoRegister;
bool AllowOverlap = false;
llvm::SmallBitVector RegMask;
llvm::SmallBitVector Free;
llvm::SmallBitVector PrecoloredUnhandledMask; // Note: only used for dumping
llvm::SmallVector<RegWeight, REGS_SIZE> Weights;
};
void initForGlobal(); void initForGlobal();
void initForInfOnly(); void initForInfOnly();
/// Free up a register for infinite-weight Cur by spilling and reloading some /// Move an item from the From set to the To set. From[Index] is pushed onto
/// register that isn't used during Cur's live range. /// the end of To[], then the item is efficiently removed from From[] by
void addSpillFill(Variable *Cur, llvm::SmallBitVector RegMask); /// effectively swapping it with the last item in From[] and then popping it
/// Move an item from the From set to the To set. From[Index] is /// from the back. As such, the caller is best off iterating over From[] in
/// pushed onto the end of To[], then the item is efficiently removed /// reverse order to avoid the need for special handling of the iterator.
/// from From[] by effectively swapping it with the last item in
/// From[] and then popping it from the back. As such, the caller is
/// best off iterating over From[] in reverse order to avoid the need
/// for special handling of the iterator.
void moveItem(UnorderedRanges &From, SizeT Index, UnorderedRanges &To) { void moveItem(UnorderedRanges &From, SizeT Index, UnorderedRanges &To) {
To.push_back(From[Index]); To.push_back(From[Index]);
From[Index] = From.back(); From[Index] = From.back();
From.pop_back(); From.pop_back();
} }
/// \name scan helper functions.
/// @{
/// Free up a register for infinite-weight Cur by spilling and reloading some
/// register that isn't used during Cur's live range.
void addSpillFill(IterationState &Iter);
/// Check for active ranges that have expired or become inactive.
void handleActiveRangeExpiredOrInactive(const Variable *Cur);
/// Check for inactive ranges that have expired or reactivated.
void handleInactiveRangeExpiredOrReactivated(const Variable *Cur);
void findRegisterPreference(IterationState &Iter);
void filterFreeWithInactiveRanges(IterationState &Iter);
void filterFreeWithPrecoloredRanges(IterationState &Iter);
void allocatePrecoloredRegister(Variable *Cur);
void allocatePreferredRegister(IterationState &Iter);
void allocateFreeRegister(IterationState &Iter);
void handleNoFreeRegisters(IterationState &Iter);
void assignFinalRegisters(const llvm::SmallBitVector &RegMaskFull,
const llvm::SmallBitVector &PreDefinedRegisters,
bool Randomized);
/// @}
void dumpLiveRangeTrace(const char *Label, const Variable *Item);
Cfg *const Func; Cfg *const Func;
GlobalContext *const Ctx;
OrderedRanges Unhandled; OrderedRanges Unhandled;
/// UnhandledPrecolored is a subset of Unhandled, specially collected /// UnhandledPrecolored is a subset of Unhandled, specially collected for
/// for faster processing. /// faster processing.
OrderedRanges UnhandledPrecolored; OrderedRanges UnhandledPrecolored;
UnorderedRanges Active, Inactive, Handled; UnorderedRanges Active, Inactive, Handled;
std::vector<InstNumberT> Kills; std::vector<InstNumberT> Kills;
RegAllocKind Kind = RAK_Unknown; RegAllocKind Kind = RAK_Unknown;
/// RegUses[I] is the number of live ranges (variables) that register I is
/// currently assigned to. It can be greater than 1 as a result of
/// AllowOverlap inference.
llvm::SmallVector<int32_t, REGS_SIZE> RegUses;
bool FindPreference = false; bool FindPreference = false;
bool FindOverlap = false; bool FindOverlap = false;
const bool Verbose;
}; };
} // end of namespace Ice } // end of namespace Ice
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment