Subzero: Improve performance of liveness analysis and live range construction.

src/IceCfg.cpp

@@ -205,9 +205,9 @@ void Cfg::liveness(LivenessMode Mode) {
   // Collect timing for just the portion that constructs the live
   // range intervals based on the end-of-live-range computation, for a
   // finer breakdown of the cost.
+  TimerMarker T1(TimerStack::TT_liveRange, this);
   // Make a final pass over instructions to delete dead instructions
   // and build each Variable's live range.
-  TimerMarker T1(TimerStack::TT_liveRange, this);
   for (CfgNode *Node : Nodes)
     Node->livenessPostprocess(Mode, getLiveness());
   if (Mode == Liveness_Intervals) {
@@ -246,7 +246,6 @@ void Cfg::liveness(LivenessMode Mode) {
       if (Var->getWeight().isInf())
         Var->setLiveRangeInfiniteWeight();
     }
-    dump();
   }
 }
 
@@ -257,24 +256,37 @@ bool Cfg::validateLiveness() const {
   bool Valid = true;
   Ostream &Str = Ctx->getStrDump();
   for (CfgNode *Node : Nodes) {
+    Inst *FirstInst = NULL;
     for (Inst *Inst : Node->getInsts()) {
       if (Inst->isDeleted())
         continue;
       if (llvm::isa<InstFakeKill>(Inst))
         continue;
+      if (FirstInst == NULL)
+        FirstInst = Inst;
       InstNumberT InstNumber = Inst->getNumber();
-      Variable *Dest = Inst->getDest();
-      if (Dest) {
-        // TODO: This instruction should actually begin Dest's live
-        // range, so we could probably test that this instruction is
-        // the beginning of some segment of Dest's live range.  But
-        // this wouldn't work with non-SSA temporaries during
-        // lowering.
-        if (!Dest->getLiveRange().containsValue(InstNumber)) {
-          Valid = false;
-          Str << "Liveness error: inst " << Inst->getNumber() << " dest ";
-          Dest->dump(this);
-          Str << " live range " << Dest->getLiveRange() << "\n";
+      if (Variable *Dest = Inst->getDest()) {
+        if (!Dest->getIgnoreLiveness()) {
+          bool Invalid = false;
+          const bool IsDest = true;
+          if (!Dest->getLiveRange().containsValue(InstNumber, IsDest))
+            Invalid = true;
+          // Check that this instruction actually *begins* Dest's live
+          // range, by checking that Dest is not live in the previous
+          // instruction.  As a special exception, we don't check this
+          // for the first instruction of the block, because a Phi
+          // temporary may be live at the end of the previous block,
+          // and if it is also assigned in the first instruction of
+          // this block, the adjacent live ranges get merged.
+          if (Inst != FirstInst && !Inst->isDestNonKillable() &&
+              Dest->getLiveRange().containsValue(InstNumber - 1, IsDest))
+            Invalid = true;
+          if (Invalid) {
+            Valid = false;
+            Str << "Liveness error: inst " << Inst->getNumber() << " dest ";
+            Dest->dump(this);
+            Str << " live range " << Dest->getLiveRange() << "\n";
+          }
         }
       }
       for (SizeT I = 0; I < Inst->getSrcSize(); ++I) {
@@ -282,7 +294,9 @@ bool Cfg::validateLiveness() const {
         SizeT NumVars = Src->getNumVars();
         for (SizeT J = 0; J < NumVars; ++J) {
           const Variable *Var = Src->getVar(J);
-          if (!Var->getLiveRange().containsValue(InstNumber)) {
+          const bool IsDest = false;
+          if (!Var->getIgnoreLiveness() &&
+              !Var->getLiveRange().containsValue(InstNumber, IsDest)) {
             Valid = false;
             Str << "Liveness error: inst " << Inst->getNumber() << " var ";
             Var->dump(this);

src/IceCfg.h

@@ -62,6 +62,7 @@ public:
 
   // Manage instruction numbering.
   InstNumberT newInstNumber() { return NextInstNumber++; }
+  InstNumberT getNextInstNumber() const { return NextInstNumber; }
 
   // Manage Variables.
   // Create a new Variable with a particular type and an optional

src/IceCfgNode.cpp

@@ -22,7 +22,8 @@
 namespace Ice {
 
 CfgNode::CfgNode(Cfg *Func, SizeT LabelNumber, IceString Name)
-    : Func(Func), Number(LabelNumber), Name(Name), HasReturn(false) {}
+    : Func(Func), Number(LabelNumber), Name(Name), HasReturn(false),
+      InstCountEstimate(0) {}
 
 // Returns the name the node was created with.  If no name was given,
 // it synthesizes a (hopefully) unique name.
@@ -38,6 +39,7 @@ IceString CfgNode::getName() const {
 // instruction list.  Validates that all Phis are added before all
 // regular instructions.
 void CfgNode::appendInst(Inst *Inst) {
+  ++InstCountEstimate;
   if (InstPhi *Phi = llvm::dyn_cast<InstPhi>(Inst)) {
     if (!Insts.empty()) {
       Func->setError("Phi instruction added to the middle of a block");
@@ -55,10 +57,12 @@ void CfgNode::appendInst(Inst *Inst) {
 // instruction numbers in a block, from lowest to highest, must not
 // overlap with the range of any other block.
 void CfgNode::renumberInstructions() {
+  InstNumberT FirstNumber = Func->getNextInstNumber();
   for (InstPhi *I : Phis)
     I->renumber(Func);
   for (Inst *I : Insts)
     I->renumber(Func);
+  InstCountEstimate = Func->getNextInstNumber() - FirstNumber;
 }
 
 // When a node is created, the OutEdges are immediately knows, but the
@@ -251,7 +255,7 @@ void CfgNode::genCode() {
 
 void CfgNode::livenessLightweight() {
   SizeT NumVars = Func->getNumVariables();
-  llvm::BitVector Live(NumVars);
+  LivenessBV Live(NumVars);
   // Process regular instructions in reverse order.
   // TODO(stichnot): Use llvm::make_range with LLVM 3.5.
   for (auto I = Insts.rbegin(), E = Insts.rend(); I != E; ++I) {
@@ -272,14 +276,21 @@ void CfgNode::livenessLightweight() {
 // again.)
 bool CfgNode::liveness(Liveness *Liveness) {
   SizeT NumVars = Liveness->getNumVarsInNode(this);
-  llvm::BitVector Live(NumVars);
+  LivenessBV Live(NumVars);
+  LiveBeginEndMap *LiveBegin = NULL;
+  LiveBeginEndMap *LiveEnd = NULL;
   // Mark the beginning and ending of each variable's live range
   // with the sentinel instruction number 0.
-  std::vector<InstNumberT> &LiveBegin = Liveness->getLiveBegin(this);
-  std::vector<InstNumberT> &LiveEnd = Liveness->getLiveEnd(this);
-  InstNumberT Sentinel = Inst::NumberSentinel;
-  LiveBegin.assign(NumVars, Sentinel);
-  LiveEnd.assign(NumVars, Sentinel);
+  if (Liveness->getMode() == Liveness_Intervals) {
+    LiveBegin = Liveness->getLiveBegin(this);
+    LiveEnd = Liveness->getLiveEnd(this);
+    LiveBegin->clear();
+    LiveEnd->clear();
+    // Guess that the number of live ranges beginning is roughly the
+    // number of instructions, and same for live ranges ending.
+    LiveBegin->reserve(getInstCountEstimate());
+    LiveEnd->reserve(getInstCountEstimate());
+  }
   // Initialize Live to be the union of all successors' LiveIn.
   for (CfgNode *Succ : OutEdges) {
     Live |= Liveness->getLiveIn(Succ);
@@ -294,7 +305,7 @@ bool CfgNode::liveness(Liveness *Liveness) {
   for (auto I = Insts.rbegin(), E = Insts.rend(); I != E; ++I) {
     if ((*I)->isDeleted())
       continue;
-    (*I)->liveness((*I)->getNumber(), Live, Liveness, this);
+    (*I)->liveness((*I)->getNumber(), Live, Liveness, LiveBegin, LiveEnd);
   }
   // Process phis in forward order so that we can override the
   // instruction number to be that of the earliest phi instruction in
@@ -305,7 +316,7 @@ bool CfgNode::liveness(Liveness *Liveness) {
       continue;
     if (FirstPhiNumber == Inst::NumberSentinel)
       FirstPhiNumber = I->getNumber();
-    I->liveness(FirstPhiNumber, Live, Liveness, this);
+    I->liveness(FirstPhiNumber, Live, Liveness, LiveBegin, LiveEnd);
   }
 
   // When using the sparse representation, after traversing the
@@ -314,7 +325,7 @@ bool CfgNode::liveness(Liveness *Liveness) {
   // by shrinking the Live vector and then testing it against the
   // pre-shrunk version.  (The shrinking is required, but the
   // validation is not.)
-  llvm::BitVector LiveOrig = Live;
+  LivenessBV LiveOrig = Live;
   Live.resize(Liveness->getNumGlobalVars());
   // Non-global arguments in the entry node are allowed to be live on
   // entry.
@@ -336,7 +347,7 @@ bool CfgNode::liveness(Liveness *Liveness) {
   }
 
   bool Changed = false;
-  llvm::BitVector &LiveIn = Liveness->getLiveIn(this);
+  LivenessBV &LiveIn = Liveness->getLiveIn(this);
   // Add in current LiveIn
   Live |= LiveIn;
   // Check result, set LiveIn=Live
@@ -346,6 +357,16 @@ bool CfgNode::liveness(Liveness *Liveness) {
   return Changed;
 }
 
+#ifndef NDEBUG
+namespace {
+
+bool comparePair(const LiveBeginEndMapEntry &A, const LiveBeginEndMapEntry &B) {
+  return A.first == B.first;
+}
+
+} // end of anonymous namespace
+#endif // NDEBUG
+
 // Now that basic liveness is complete, remove dead instructions that
 // were tentatively marked as dead, and compute actual live ranges.
 // It is assumed that within a single basic block, a live range begins
@@ -396,34 +417,63 @@ void CfgNode::livenessPostprocess(LivenessMode Mode, Liveness *Liveness) {
   TimerMarker T1(TimerStack::TT_liveRangeCtor, Func);
 
   SizeT NumVars = Liveness->getNumVarsInNode(this);
-  SizeT NumGlobals = Liveness->getNumGlobalVars();
-  llvm::BitVector &LiveIn = Liveness->getLiveIn(this);
-  llvm::BitVector &LiveOut = Liveness->getLiveOut(this);
-  std::vector<InstNumberT> &LiveBegin = Liveness->getLiveBegin(this);
-  std::vector<InstNumberT> &LiveEnd = Liveness->getLiveEnd(this);
-  for (SizeT i = 0; i < NumVars; ++i) {
-    // Deal with the case where the variable is both live-in and
-    // live-out, but LiveEnd comes before LiveBegin.  In this case, we
-    // need to add two segments to the live range because there is a
-    // hole in the middle.  This would typically happen as a result of
-    // phi lowering in the presence of loopback edges.
-    bool IsGlobal = (i < NumGlobals);
-    if (IsGlobal && LiveIn[i] && LiveOut[i] && LiveBegin[i] > LiveEnd[i]) {
-      Variable *Var = Liveness->getVariable(i, this);
-      Liveness->addLiveRange(Var, FirstInstNum, LiveEnd[i], 1);
-      Liveness->addLiveRange(Var, LiveBegin[i], LastInstNum + 1, 1);
-      continue;
+  LivenessBV &LiveIn = Liveness->getLiveIn(this);
+  LivenessBV &LiveOut = Liveness->getLiveOut(this);
+  LiveBeginEndMap &MapBegin = *Liveness->getLiveBegin(this);
+  LiveBeginEndMap &MapEnd = *Liveness->getLiveEnd(this);
+  std::sort(MapBegin.begin(), MapBegin.end());
+  std::sort(MapEnd.begin(), MapEnd.end());
+  // Verify there are no duplicates.
+  assert(std::adjacent_find(MapBegin.begin(), MapBegin.end(), comparePair) ==
+         MapBegin.end());
+  assert(std::adjacent_find(MapEnd.begin(), MapEnd.end(), comparePair) ==
+         MapEnd.end());
+
+  LivenessBV LiveInAndOut = LiveIn;
+  LiveInAndOut &= LiveOut;
+
+  // Iterate in parallel across the sorted MapBegin[] and MapEnd[].
+  auto IBB = MapBegin.begin(), IEB = MapEnd.begin();
+  auto IBE = MapBegin.end(), IEE = MapEnd.end();
+  while (IBB != IBE || IEB != IEE) {
+    SizeT i1 = IBB == IBE ? NumVars : IBB->first;
+    SizeT i2 = IEB == IEE ? NumVars : IEB->first;
+    SizeT i = std::min(i1, i2);
+    // i1 is the Variable number of the next MapBegin entry, and i2 is
+    // the Variable number of the next MapEnd entry.  If i1==i2, then
+    // the Variable's live range begins and ends in this block.  If
+    // i1<i2, then i1's live range begins at instruction IBB->second
+    // and extends through the end of the block.  If i1>i2, then i2's
+    // live range begins at the first instruction of the block and
+    // ends at IEB->second.  In any case, we choose the lesser of i1
+    // and i2 and proceed accordingly.
+    InstNumberT LB = i == i1 ? IBB->second : FirstInstNum;
+    InstNumberT LE = i == i2 ? IEB->second : LastInstNum + 1;
+
+    Variable *Var = Liveness->getVariable(i, this);
+    if (!Var->getIgnoreLiveness()) {
+      if (LB > LE) {
+        Liveness->addLiveRange(Var, FirstInstNum, LE, 1);
+        Liveness->addLiveRange(Var, LB, LastInstNum + 1, 1);
+        // Assert that Var is a global variable by checking that its
+        // liveness index is less than the number of globals.  This
+        // ensures that the LiveInAndOut[] access is valid.
+        assert(i < Liveness->getNumGlobalVars());
+        LiveInAndOut[i] = false;
+      } else {
+        Liveness->addLiveRange(Var, LB, LE, 1);
+      }
     }
-    InstNumberT Begin = (IsGlobal && LiveIn[i]) ? FirstInstNum : LiveBegin[i];
-    InstNumberT End = (IsGlobal && LiveOut[i]) ? LastInstNum + 1 : LiveEnd[i];
-    if (Begin == Inst::NumberSentinel && End == Inst::NumberSentinel)
-      continue;
-    if (Begin <= FirstInstNum)
-      Begin = FirstInstNum;
-    if (End == Inst::NumberSentinel)
-      End = LastInstNum + 1;
+    if (i == i1)
+      ++IBB;
+    if (i == i2)
+      ++IEB;
+  }
+  // Process the variables that are live across the entire block.
+  for (int i = LiveInAndOut.find_first(); i != -1;
+       i = LiveInAndOut.find_next(i)) {
     Variable *Var = Liveness->getVariable(i, this);
-    Liveness->addLiveRange(Var, Begin, End, 1);
+    Liveness->addLiveRange(Var, FirstInstNum, LastInstNum + 1, 1);
   }
 }
 
@@ -504,7 +554,7 @@ void CfgNode::dump(Cfg *Func) const {
     Str << "\n";
   }
   // Dump the live-in variables.
-  llvm::BitVector LiveIn;
+  LivenessBV LiveIn;
   if (Liveness)
     LiveIn = Liveness->getLiveIn(this);
   if (Func->getContext()->isVerbose(IceV_Liveness) && !LiveIn.empty()) {
@@ -524,7 +574,7 @@ void CfgNode::dump(Cfg *Func) const {
       I->dumpDecorated(Func);
   }
   // Dump the live-out variables.
-  llvm::BitVector LiveOut;
+  LivenessBV LiveOut;
   if (Liveness)
     LiveOut = Liveness->getLiveOut(this);
   if (Func->getContext()->isVerbose(IceV_Liveness) && !LiveOut.empty()) {

src/IceCfgNode.h

@@ -51,6 +51,11 @@ public:
   InstList &getInsts() { return Insts; }
   void appendInst(Inst *Inst);
   void renumberInstructions();
+  // Rough and generally conservative estimate of the number of
+  // instructions in the block.  It is updated when an instruction is
+  // added, but not when deleted.  It is recomputed during
+  // renumberInstructions().
+  InstNumberT getInstCountEstimate() const { return InstCountEstimate; }
 
   // Add a predecessor edge to the InEdges list for each of this
   // node's successors.
@@ -77,6 +82,7 @@ private:
   const SizeT Number; // label index
   IceString Name;     // for dumping only
   bool HasReturn;     // does this block need an epilog?
+  InstNumberT InstCountEstimate; // rough instruction count estimate
   NodeList InEdges;   // in no particular order
   NodeList OutEdges;  // in no particular order
   PhiList Phis;       // unordered set of phi instructions

src/IceDefs.h

@@ -71,6 +71,13 @@ typedef uint32_t SizeT;
 // numbers are used for representing Variable live ranges.
 typedef int32_t InstNumberT;
 
+// A LiveBeginEndMapEntry maps a Variable::Number value to an
+// Inst::Number value, giving the instruction number that begins or
+// ends a variable's live range.
+typedef std::pair<SizeT, InstNumberT> LiveBeginEndMapEntry;
+typedef std::vector<LiveBeginEndMapEntry> LiveBeginEndMap;
+typedef llvm::BitVector LivenessBV;
+
 typedef uint32_t TimerStackIdT;
 typedef uint32_t TimerIdT;
 

src/IceInst.cpp

@@ -71,7 +71,8 @@ const struct InstIcmpAttributes_ {
 
 Inst::Inst(Cfg *Func, InstKind Kind, SizeT MaxSrcs, Variable *Dest)
     : Kind(Kind), Number(Func->newInstNumber()), Deleted(false), Dead(false),
-      HasSideEffects(false), Dest(Dest), MaxSrcs(MaxSrcs), NumSrcs(0),
+      HasSideEffects(false), IsDestNonKillable(false), Dest(Dest),
+      MaxSrcs(MaxSrcs), NumSrcs(0),
       Srcs(Func->allocateArrayOf<Operand *>(MaxSrcs)), LiveRangesEnded(0) {}
 
 // Assign the instruction a new number.
@@ -111,18 +112,19 @@ bool Inst::isLastUse(const Operand *TestSrc) const {
   return false;
 }
 
-void Inst::livenessLightweight(Cfg *Func, llvm::BitVector &Live) {
+void Inst::livenessLightweight(Cfg *Func, LivenessBV &Live) {
   assert(!isDeleted());
   if (llvm::isa<InstFakeKill>(this))
     return;
   resetLastUses();
+  VariablesMetadata *VMetadata = Func->getVMetadata();
   SizeT VarIndex = 0;
   for (SizeT I = 0; I < getSrcSize(); ++I) {
     Operand *Src = getSrc(I);
     SizeT NumVars = Src->getNumVars();
     for (SizeT J = 0; J < NumVars; ++J, ++VarIndex) {
       const Variable *Var = Src->getVar(J);
-      if (Func->getVMetadata()->isMultiBlock(Var))
+      if (VMetadata->isMultiBlock(Var))
         continue;
       SizeT Index = Var->getIndex();
       if (Live[Index])
@@ -133,20 +135,23 @@ void Inst::livenessLightweight(Cfg *Func, llvm::BitVector &Live) {
   }
 }
 
-void Inst::liveness(InstNumberT InstNumber, llvm::BitVector &Live,
-                    Liveness *Liveness, const CfgNode *Node) {
+void Inst::liveness(InstNumberT InstNumber, LivenessBV &Live,
+                    Liveness *Liveness, LiveBeginEndMap *LiveBegin,
+                    LiveBeginEndMap *LiveEnd) {
   assert(!isDeleted());
   if (llvm::isa<InstFakeKill>(this))
     return;
 
-  std::vector<InstNumberT> &LiveBegin = Liveness->getLiveBegin(Node);
-  std::vector<InstNumberT> &LiveEnd = Liveness->getLiveEnd(Node);
   Dead = false;
   if (Dest) {
-    SizeT VarNum = Liveness->getLiveIndex(Dest);
+    SizeT VarNum = Liveness->getLiveIndex(Dest->getIndex());
     if (Live[VarNum]) {
-      Live[VarNum] = false;
-      LiveBegin[VarNum] = InstNumber;
+      if (!isDestNonKillable()) {
+        Live[VarNum] = false;
+        if (LiveBegin) {
+          LiveBegin->push_back(std::make_pair(VarNum, InstNumber));
+        }
+      }
     } else {
       if (!hasSideEffects())
         Dead = true;
@@ -164,7 +169,7 @@ void Inst::liveness(InstNumberT InstNumber, llvm::BitVector &Live,
     SizeT NumVars = Src->getNumVars();
     for (SizeT J = 0; J < NumVars; ++J, ++VarIndex) {
       const Variable *Var = Src->getVar(J);
-      SizeT VarNum = Liveness->getLiveIndex(Var);
+      SizeT VarNum = Liveness->getLiveIndex(Var->getIndex());
       if (!Live[VarNum]) {
         setLastUse(VarIndex);
         if (!IsPhi) {
@@ -185,8 +190,13 @@ void Inst::liveness(InstNumberT InstNumber, llvm::BitVector &Live,
           // setting it only when LiveEnd[VarNum]==0 (sentinel value).
           // Note that it's OK to set LiveBegin multiple times because
           // of the backwards traversal.
-          if (LiveEnd[VarNum] == 0) {
-            LiveEnd[VarNum] = InstNumber;
+          if (LiveEnd) {
+            // Ideally, we would verify that VarNum wasn't already
+            // added in this block, but this can't be done very
+            // efficiently with LiveEnd as a vector.  Instead,
+            // livenessPostprocess() verifies this after the vector
+            // has been sorted.
+            LiveEnd->push_back(std::make_pair(VarNum, InstNumber));
           }
         }
       }
@@ -320,14 +330,14 @@ Operand *InstPhi::getOperandForTarget(CfgNode *Target) const {
 // Updates liveness for a particular operand based on the given
 // predecessor edge.  Doesn't mark the operand as live if the Phi
 // instruction is dead or deleted.
-void InstPhi::livenessPhiOperand(llvm::BitVector &Live, CfgNode *Target,
+void InstPhi::livenessPhiOperand(LivenessBV &Live, CfgNode *Target,
                                  Liveness *Liveness) {
   if (isDeleted() || Dead)
     return;
   for (SizeT I = 0; I < getSrcSize(); ++I) {
     if (Labels[I] == Target) {
       if (Variable *Var = llvm::dyn_cast<Variable>(getSrc(I))) {
-        SizeT SrcIndex = Liveness->getLiveIndex(Var);
+        SizeT SrcIndex = Liveness->getLiveIndex(Var->getIndex());
         if (!Live[SrcIndex]) {
           setLastUse(I);
           Live[SrcIndex] = true;

src/IceInst.h

@@ -66,8 +66,7 @@ public:
 
   InstNumberT getNumber() const { return Number; }
   void renumber(Cfg *Func);
-  static const InstNumberT NumberDeleted = -1;
-  static const InstNumberT NumberSentinel = 0;
+  enum { NumberDeleted = -1, NumberSentinel = 0 };
 
   bool isDeleted() const { return Deleted; }
   void setDeleted() { Deleted = true; }
@@ -75,6 +74,9 @@ public:
 
   bool hasSideEffects() const { return HasSideEffects; }
 
+  bool isDestNonKillable() const { return IsDestNonKillable; }
+  void setDestNonKillable() { IsDestNonKillable = true; }
+
   Variable *getDest() const { return Dest; }
 
   SizeT getSrcSize() const { return NumSrcs; }
@@ -98,9 +100,9 @@ public:
 
   virtual bool isSimpleAssign() const { return false; }
 
-  void livenessLightweight(Cfg *Func, llvm::BitVector &Live);
-  void liveness(InstNumberT InstNumber, llvm::BitVector &Live,
-                Liveness *Liveness, const CfgNode *Node);
+  void livenessLightweight(Cfg *Func, LivenessBV &Live);
+  void liveness(InstNumberT InstNumber, LivenessBV &Live, Liveness *Liveness,
+                LiveBeginEndMap *LiveBegin, LiveBeginEndMap *LiveEnd);
 
   // Get the number of native instructions that this instruction
   // ultimately emits.  By default, high-level instructions don't
@@ -146,6 +148,10 @@ protected:
   // call or a volatile load that can't be removed even if its Dest
   // variable is not live.
   bool HasSideEffects;
+  // IsDestNonKillable means that liveness analysis shouldn't consider
+  // this instruction to kill the Dest variable.  This is used when
+  // lowering produces two assignments to the same variable.
+  bool IsDestNonKillable;
 
   Variable *Dest;
   const SizeT MaxSrcs; // only used for assert
@@ -534,7 +540,7 @@ public:
   }
   void addArgument(Operand *Source, CfgNode *Label);
   Operand *getOperandForTarget(CfgNode *Target) const;
-  void livenessPhiOperand(llvm::BitVector &Live, CfgNode *Target,
+  void livenessPhiOperand(LivenessBV &Live, CfgNode *Target,
                           Liveness *Liveness);
   Inst *lower(Cfg *Func);
   void dump(const Cfg *Func) const override;

src/IceLiveness.cpp

@@ -32,6 +32,7 @@ void Liveness::init() {
   // Initialize most of the container sizes.
   SizeT NumVars = Func->getVariables().size();
   SizeT NumNodes = Func->getNumNodes();
+  VariablesMetadata *VMetadata = Func->getVMetadata();
   Nodes.resize(NumNodes);
   VarToLiveMap.resize(NumVars);
   if (Mode == Liveness_Intervals)
@@ -41,10 +42,10 @@ void Liveness::init() {
   // block.
   for (SizeT i = 0; i < NumVars; ++i) {
     Variable *Var = Func->getVariables()[i];
-    if (Func->getVMetadata()->isMultiBlock(Var)) {
+    if (VMetadata->isMultiBlock(Var)) {
       ++NumGlobals;
     } else {
-      SizeT Index = Func->getVMetadata()->getLocalUseNode(Var)->getIndex();
+      SizeT Index = VMetadata->getLocalUseNode(Var)->getIndex();
       ++Nodes[Index].NumLocals;
     }
   }
@@ -64,11 +65,11 @@ void Liveness::init() {
     Variable *Var = Func->getVariables()[i];
     SizeT VarIndex = Var->getIndex();
     SizeT LiveIndex;
-    if (Func->getVMetadata()->isMultiBlock(Var)) {
+    if (VMetadata->isMultiBlock(Var)) {
       LiveIndex = TmpNumGlobals++;
       LiveToVarMap[LiveIndex] = Var;
     } else {
-      SizeT NodeIndex = Func->getVMetadata()->getLocalUseNode(Var)->getIndex();
+      SizeT NodeIndex = VMetadata->getLocalUseNode(Var)->getIndex();
       LiveIndex = Nodes[NodeIndex].NumLocals++;
       Nodes[NodeIndex].LiveToVarMap[LiveIndex] = Var;
       LiveIndex += NumGlobals;
@@ -97,10 +98,6 @@ Variable *Liveness::getVariable(SizeT LiveIndex, const CfgNode *Node) const {
   return Nodes[NodeIndex].LiveToVarMap[LiveIndex - NumGlobals];
 }
 
-SizeT Liveness::getLiveIndex(const Variable *Var) const {
-  return VarToLiveMap[Var->getIndex()];
-}
-
 void Liveness::addLiveRange(Variable *Var, InstNumberT Start, InstNumberT End,
                             uint32_t WeightDelta) {
   LiveRange &LiveRange = LiveRanges[Var->getIndex()];

src/IceLiveness.h

@@ -37,11 +37,12 @@ public:
   // LiveIn and LiveOut track the in- and out-liveness of the global
   // variables.  The size of each vector is
   // LivenessNode::NumGlobals.
-  llvm::BitVector LiveIn, LiveOut;
+  LivenessBV LiveIn, LiveOut;
   // LiveBegin and LiveEnd track the instruction numbers of the start
   // and end of each variable's live range within this block.  The
-  // size of each vector is NumLocals + Liveness::NumGlobals.
-  std::vector<InstNumberT> LiveBegin, LiveEnd;
+  // index/key of each element is less than NumLocals +
+  // Liveness::NumGlobals.
+  LiveBeginEndMap LiveBegin, LiveEnd;
 
 private:
   // TODO: Disable these constructors when Liveness::Nodes is no
@@ -55,23 +56,25 @@ public:
   Liveness(Cfg *Func, LivenessMode Mode)
       : Func(Func), Mode(Mode), NumGlobals(0) {}
   void init();
+  Cfg *getFunc() const { return Func; }
+  LivenessMode getMode() const { return Mode; }
   Variable *getVariable(SizeT LiveIndex, const CfgNode *Node) const;
-  SizeT getLiveIndex(const Variable *Var) const;
+  SizeT getLiveIndex(SizeT VarIndex) const { return VarToLiveMap[VarIndex]; }
   SizeT getNumGlobalVars() const { return NumGlobals; }
   SizeT getNumVarsInNode(const CfgNode *Node) const {
     return NumGlobals + Nodes[Node->getIndex()].NumLocals;
   }
-  llvm::BitVector &getLiveIn(const CfgNode *Node) {
+  LivenessBV &getLiveIn(const CfgNode *Node) {
     return Nodes[Node->getIndex()].LiveIn;
   }
-  llvm::BitVector &getLiveOut(const CfgNode *Node) {
+  LivenessBV &getLiveOut(const CfgNode *Node) {
     return Nodes[Node->getIndex()].LiveOut;
   }
-  std::vector<InstNumberT> &getLiveBegin(const CfgNode *Node) {
-    return Nodes[Node->getIndex()].LiveBegin;
+  LiveBeginEndMap *getLiveBegin(const CfgNode *Node) {
+    return &Nodes[Node->getIndex()].LiveBegin;
   }
-  std::vector<InstNumberT> &getLiveEnd(const CfgNode *Node) {
-    return Nodes[Node->getIndex()].LiveEnd;
+  LiveBeginEndMap *getLiveEnd(const CfgNode *Node) {
+    return &Nodes[Node->getIndex()].LiveEnd;
   }
   LiveRange &getLiveRange(Variable *Var);
   void addLiveRange(Variable *Var, InstNumberT Start, InstNumberT End,

src/IceOperand.cpp

@@ -151,10 +151,13 @@ bool LiveRange::overlapsInst(InstNumberT OtherBegin, bool UseTrimmed) const {
 
 // Returns true if the live range contains the given instruction
 // number.  This is only used for validating the live range
-// calculation.
-bool LiveRange::containsValue(InstNumberT Value) const {
+// calculation.  The IsDest argument indicates whether the Variable
+// being tested is used in the Dest position (as opposed to a Src
+// position).
+bool LiveRange::containsValue(InstNumberT Value, bool IsDest) const {
   for (const RangeElementType &I : Range) {
-    if (I.first <= Value && Value <= I.second)
+    if (I.first <= Value &&
+        (Value < I.second || (!IsDest && Value == I.second)))
       return true;
   }
   return false;
@@ -184,6 +187,8 @@ Variable Variable::asType(Type Ty) {
 
 void VariableTracking::markUse(const Inst *Instr, const CfgNode *Node,
                                bool IsFromDef, bool IsImplicit) {
+  if (MultiBlock == MBS_MultiBlock)
+    return;
   // TODO(stichnot): If the use occurs as a source operand in the
   // first instruction of the block, and its definition is in this
   // block's only predecessor, we might consider not marking this as a
@@ -301,9 +306,7 @@ void VariablesMetadata::init() {
     Metadata[Var->getIndex()].markUse(NoInst, EntryNode, IsFromDef, IsImplicit);
   }
 
-  SizeT NumNodes = Func->getNumNodes();
-  for (SizeT N = 0; N < NumNodes; ++N) {
-    CfgNode *Node = Func->getNodes()[N];
+  for (CfgNode *Node : Func->getNodes()) {
     for (Inst *I : Node->getInsts()) {
       if (I->isDeleted())
         continue;

src/IceOperand.h

@@ -314,7 +314,7 @@ public:
   bool endsBefore(const LiveRange &Other) const;
   bool overlaps(const LiveRange &Other, bool UseTrimmed = false) const;
   bool overlapsInst(InstNumberT OtherBegin, bool UseTrimmed = false) const;
-  bool containsValue(InstNumberT Value) const;
+  bool containsValue(InstNumberT Value, bool IsDest) const;
   bool isEmpty() const { return Range.empty(); }
   bool isNonpoints() const { return IsNonpoints; }
   InstNumberT getStart() const {
@@ -390,6 +390,9 @@ public:
   bool getIsImplicitArg() const { return IsImplicitArgument; }
   void setIsImplicitArg(bool Val = true) { IsImplicitArgument = Val; }
 
+  void setIgnoreLiveness() { IgnoreLiveness = true; }
+  bool getIgnoreLiveness() const { return IgnoreLiveness; }
+
   int32_t getStackOffset() const { return StackOffset; }
   void setStackOffset(int32_t Offset) { StackOffset = Offset; }
 
@@ -454,8 +457,9 @@ public:
 protected:
   Variable(OperandKind K, Type Ty, SizeT Index, const IceString &Name)
       : Operand(K, Ty), Number(Index), Name(Name), IsArgument(false),
-        IsImplicitArgument(false), StackOffset(0), RegNum(NoRegister),
-        RegNumTmp(NoRegister), Weight(1), LoVar(NULL), HiVar(NULL) {
+        IsImplicitArgument(false), IgnoreLiveness(false), StackOffset(0),
+        RegNum(NoRegister), RegNumTmp(NoRegister), Weight(1), LoVar(NULL),
+        HiVar(NULL) {
     Vars = VarsReal;
     Vars[0] = this;
     NumVars = 1;
@@ -467,6 +471,10 @@ protected:
   IceString Name;
   bool IsArgument;
   bool IsImplicitArgument;
+  // IgnoreLiveness means that the variable should be ignored when
+  // constructing and validating live ranges.  This is usually
+  // reserved for the stack pointer.
+  bool IgnoreLiveness;
   // StackOffset is the canonical location on stack (only if
   // RegNum==NoRegister || IsArgument).
   int32_t StackOffset;

src/IceTargetLoweringX8632.cpp

@@ -453,8 +453,10 @@ Variable *TargetX8632::getPhysicalRegister(SizeT RegNum) {
     PhysicalRegisters[RegNum] = Reg;
     // Specially mark esp as an "argument" so that it is considered
     // live upon function entry.
-    if (RegNum == RegX8632::Reg_esp)
+    if (RegNum == RegX8632::Reg_esp) {
       Func->addImplicitArg(Reg);
+      Reg->setIgnoreLiveness();
+    }
   }
   return Reg;
 }
@@ -1257,12 +1259,11 @@ void TargetX8632::lowerArithmetic(const InstArithmetic *Inst) {
       _shl(T_2, T_1);
       _test(T_1, BitTest);
       _br(CondX86::Br_e, Label);
-      // Because of the intra-block control flow, we need to fake a use
-      // of T_3 to prevent its earlier definition from being dead-code
-      // eliminated in the presence of its later definition.
-      Context.insert(InstFakeUse::create(Func, T_3));
-      _mov(T_3, T_2);
-      _mov(T_2, Zero);
+      // T_2 and T_3 are being assigned again because of the
+      // intra-block control flow, so we need the _mov_nonkillable
+      // variant to avoid liveness problems.
+      _mov_nonkillable(T_3, T_2);
+      _mov_nonkillable(T_2, Zero);
       Context.insert(Label);
       _mov(DestLo, T_2);
       _mov(DestHi, T_3);
@@ -1293,12 +1294,11 @@ void TargetX8632::lowerArithmetic(const InstArithmetic *Inst) {
       _shr(T_3, T_1);
       _test(T_1, BitTest);
       _br(CondX86::Br_e, Label);
-      // Because of the intra-block control flow, we need to fake a use
-      // of T_3 to prevent its earlier definition from being dead-code
-      // eliminated in the presence of its later definition.
-      Context.insert(InstFakeUse::create(Func, T_2));
-      _mov(T_2, T_3);
-      _mov(T_3, Zero);
+      // T_2 and T_3 are being assigned again because of the
+      // intra-block control flow, so we need the _mov_nonkillable
+      // variant to avoid liveness problems.
+      _mov_nonkillable(T_2, T_3);
+      _mov_nonkillable(T_3, Zero);
       Context.insert(Label);
       _mov(DestLo, T_2);
       _mov(DestHi, T_3);
@@ -1329,11 +1329,11 @@ void TargetX8632::lowerArithmetic(const InstArithmetic *Inst) {
       _sar(T_3, T_1);
       _test(T_1, BitTest);
       _br(CondX86::Br_e, Label);
-      // Because of the intra-block control flow, we need to fake a use
-      // of T_3 to prevent its earlier definition from being dead-code
-      // eliminated in the presence of its later definition.
-      Context.insert(InstFakeUse::create(Func, T_2));
-      _mov(T_2, T_3);
+      // T_2 and T_3 are being assigned again because of the
+      // intra-block control flow, so T_2 needs the _mov_nonkillable
+      // variant to avoid liveness problems.  T_3 doesn't need special
+      // treatment because it is reassigned via _sar instead of _mov.
+      _mov_nonkillable(T_2, T_3);
       _sar(T_3, SignExtend);
       Context.insert(Label);
       _mov(DestLo, T_2);
@@ -2516,10 +2516,9 @@ void TargetX8632::lowerFcmp(const InstFcmp *Inst) {
     if (HasC2) {
       _br(TableFcmp[Index].C2, Label);
     }
-    Context.insert(InstFakeUse::create(Func, Dest));
     Constant *NonDefault =
         Ctx->getConstantInt32(IceType_i32, !TableFcmp[Index].Default);
-    _mov(Dest, NonDefault);
+    _mov_nonkillable(Dest, NonDefault);
     Context.insert(Label);
   }
 }
@@ -2675,8 +2674,7 @@ void TargetX8632::lowerIcmp(const InstIcmp *Inst) {
       _br(CondX86::Br_ne, Label);
       _cmp(Src0HiRM, Src1HiRI);
       _br(CondX86::Br_ne, Label);
-      Context.insert(InstFakeUse::create(Func, Dest));
-      _mov(Dest, (Condition == InstIcmp::Eq ? One : Zero));
+      _mov_nonkillable(Dest, (Condition == InstIcmp::Eq ? One : Zero));
       Context.insert(Label);
     } else {
       InstX8632Label *LabelFalse = InstX8632Label::create(Func, this);
@@ -2688,8 +2686,7 @@ void TargetX8632::lowerIcmp(const InstIcmp *Inst) {
       _cmp(Src0LoRM, Src1LoRI);
       _br(TableIcmp64[Index].C3, LabelTrue);
       Context.insert(LabelFalse);
-      Context.insert(InstFakeUse::create(Func, Dest));
-      _mov(Dest, Zero);
+      _mov_nonkillable(Dest, Zero);
       Context.insert(LabelTrue);
     }
     return;
@@ -2702,8 +2699,7 @@ void TargetX8632::lowerIcmp(const InstIcmp *Inst) {
   _cmp(Src0RM, Src1);
   _mov(Dest, One);
   _br(getIcmp32Mapping(Inst->getCondition()), Label);
-  Context.insert(InstFakeUse::create(Func, Dest));
-  _mov(Dest, Zero);
+  _mov_nonkillable(Dest, Zero);
   Context.insert(Label);
 }
 
@@ -3134,7 +3130,7 @@ void TargetX8632::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
   }
   case Intrinsics::Stackrestore: {
     Variable *esp = Func->getTarget()->getPhysicalRegister(RegX8632::Reg_esp);
-    _mov(esp, Instr->getArg(0));
+    _mov_nonkillable(esp, Instr->getArg(0));
     return;
   }
   case Intrinsics::Trap:
@@ -3943,22 +3939,19 @@ void TargetX8632::lowerSelect(const InstSelect *Inst) {
     _mov(DestLo, SrcLoRI);
     _mov(DestHi, SrcHiRI);
     _br(CondX86::Br_ne, Label);
-    Context.insert(InstFakeUse::create(Func, DestLo));
-    Context.insert(InstFakeUse::create(Func, DestHi));
     Operand *SrcFLo = loOperand(SrcF);
     Operand *SrcFHi = hiOperand(SrcF);
     SrcLoRI = legalize(SrcFLo, Legal_Reg | Legal_Imm);
     SrcHiRI = legalize(SrcFHi, Legal_Reg | Legal_Imm);
-    _mov(DestLo, SrcLoRI);
-    _mov(DestHi, SrcHiRI);
+    _mov_nonkillable(DestLo, SrcLoRI);
+    _mov_nonkillable(DestHi, SrcHiRI);
   } else {
     _cmp(ConditionRM, Zero);
     SrcT = legalize(SrcT, Legal_Reg | Legal_Imm);
     _mov(Dest, SrcT);
     _br(CondX86::Br_ne, Label);
-    Context.insert(InstFakeUse::create(Func, Dest));
     SrcF = legalize(SrcF, Legal_Reg | Legal_Imm);
-    _mov(Dest, SrcF);
+    _mov_nonkillable(Dest, SrcF);
   }
 
   Context.insert(Label);
@@ -4299,9 +4292,22 @@ Variable *TargetX8632::makeReg(Type Type, int32_t RegNum) {
 }
 
 void TargetX8632::postLower() {
-  if (Ctx->getOptLevel() != Opt_m1)
+  if (Ctx->getOptLevel() != Opt_m1) {
+    // Find two-address non-SSA instructions where Dest==Src0, and set
+    // the DestNonKillable flag to keep liveness analysis consistent.
+    for (Inst *Inst : Context) {
+      if (Inst->isDeleted())
+        continue;
+      if (Variable *Dest = Inst->getDest()) {
+        // TODO(stichnot): We may need to consider all source
+        // operands, not just the first one, if using 3-address
+        // instructions.
+        if (Inst->getSrcSize() > 0 && Inst->getSrc(0) == Dest)
+          Inst->setDestNonKillable();
+      }
+    }
     return;
-  TimerMarker T(TimerStack::TT_postLower, Func);
+  }
   // TODO: Avoid recomputing WhiteList every instruction.
   RegSetMask RegInclude = RegSet_All;
   RegSetMask RegExclude = RegSet_StackPointer;

src/IceTargetLoweringX8632.h

@@ -247,6 +247,7 @@ protected:
     // Mark eax as possibly modified by cmpxchg.
     Context.insert(
         InstFakeDef::create(Func, Eax, llvm::dyn_cast<Variable>(DestOrAddr)));
+    _set_dest_nonkillable();
   }
   void _cmpxchg8b(OperandX8632Mem *Addr, Variable *Edx, Variable *Eax,
                   Variable *Ecx, Variable *Ebx, bool Locked) {
@@ -254,7 +255,9 @@ protected:
         InstX8632Cmpxchg8b::create(Func, Addr, Edx, Eax, Ecx, Ebx, Locked));
     // Mark edx, and eax as possibly modified by cmpxchg8b.
     Context.insert(InstFakeDef::create(Func, Edx));
+    _set_dest_nonkillable();
     Context.insert(InstFakeDef::create(Func, Eax));
+    _set_dest_nonkillable();
   }
   void _cvt(Variable *Dest, Operand *Src0, InstX8632Cvt::CvtVariant Variant) {
     Context.insert(InstX8632Cvt::create(Func, Dest, Src0, Variant));
@@ -294,6 +297,11 @@ protected:
       Dest = makeReg(Src0->getType(), RegNum);
     Context.insert(InstX8632Mov::create(Func, Dest, Src0));
   }
+  void _mov_nonkillable(Variable *Dest, Operand *Src0) {
+    Inst *NewInst = InstX8632Mov::create(Func, Dest, Src0);
+    NewInst->setDestNonKillable();
+    Context.insert(NewInst);
+  }
   void _movd(Variable *Dest, Operand *Src0) {
     Context.insert(InstX8632Movd::create(Func, Dest, Src0));
   }
@@ -445,16 +453,21 @@ protected:
     // Model that update with a FakeDef.
     Context.insert(
         InstFakeDef::create(Func, Src, llvm::dyn_cast<Variable>(Dest)));
+    _set_dest_nonkillable();
   }
   void _xchg(Operand *Dest, Variable *Src) {
     Context.insert(InstX8632Xchg::create(Func, Dest, Src));
     // The xchg modifies Dest and Src -- model that update with a FakeDef.
     Context.insert(
         InstFakeDef::create(Func, Src, llvm::dyn_cast<Variable>(Dest)));
+    _set_dest_nonkillable();
   }
   void _xor(Variable *Dest, Operand *Src0) {
     Context.insert(InstX8632Xor::create(Func, Dest, Src0));
   }
+  void _set_dest_nonkillable() {
+    Context.getLastInserted()->setDestNonKillable();
+  }
 
   const X86InstructionSet InstructionSet;
   bool IsEbpBasedFrame;

src/IceTimerTree.cpp

@@ -81,7 +81,6 @@ void TimerStack::update() {
   // elements and to the flat element for the top of the stack.
   double Current = timestamp();
   double Delta = Current - LastTimestamp;
-  LastTimestamp = Current;
   if (StackTop) {
     TimerIdT Leaf = Nodes[StackTop].Interior;
     if (Leaf >= LeafTimes.size())
@@ -95,6 +94,11 @@ void TimerStack::update() {
     assert(Next < Prefix);
     Prefix = Next;
   }
+  // Capture the next timestamp *after* the updates are finished.
+  // This minimizes how much the timer can perturb the reported
+  // timing.  The numbers may not sum to 100%, and the missing amount
+  // is indicative of the overhead of timing.
+  LastTimestamp = timestamp();
 }
 
 void TimerStack::reset() {

src/IceTimerTree.def

@@ -37,7 +37,6 @@
   X(parse)                 \
   X(placePhiLoads)         \
   X(placePhiStores)        \
-  X(postLower)             \
   X(regAlloc)              \
   X(renumberInstructions)  \
   X(szmain)                \

tests_lit/llvm2ice_tests/64bit.pnacl.ll

@@ -317,6 +317,24 @@ entry:
 ; OPTM1: test {{.*}}, 32
 ; OPTM1: je
 
+define internal i32 @shl64BitSignedTrunc(i64 %a, i64 %b) {
+entry:
+  %shl = shl i64 %a, %b
+  %result = trunc i64 %shl to i32
+  ret i32 %result
+}
+; CHECK-LABEL: shl64BitSignedTrunc
+; CHECK: mov
+; CHECK: shl e
+; CHECK: test {{.*}}, 32
+; CHECK: je
+;
+; OPTM1-LABEL: shl64BitSignedTrunc
+; OPTM1: shld
+; OPTM1: shl e
+; OPTM1: test {{.*}}, 32
+; OPTM1: je
+
 define internal i64 @shl64BitUnsigned(i64 %a, i64 %b) {
 entry:
   %shl = shl i64 %a, %b
@@ -353,6 +371,25 @@ entry:
 ; OPTM1: je
 ; OPTM1: sar {{.*}}, 31
 
+define internal i32 @shr64BitSignedTrunc(i64 %a, i64 %b) {
+entry:
+  %shr = ashr i64 %a, %b
+  %result = trunc i64 %shr to i32
+  ret i32 %result
+}
+; CHECK-LABEL: shr64BitSignedTrunc
+; CHECK: shrd
+; CHECK: sar
+; CHECK: test {{.*}}, 32
+; CHECK: je
+;
+; OPTM1-LABEL: shr64BitSignedTrunc
+; OPTM1: shrd
+; OPTM1: sar
+; OPTM1: test {{.*}}, 32
+; OPTM1: je
+; OPTM1: sar {{.*}}, 31
+
 define internal i64 @shr64BitUnsigned(i64 %a, i64 %b) {
 entry:
   %shr = lshr i64 %a, %b
@@ -370,6 +407,24 @@ entry:
 ; OPTM1: test {{.*}}, 32
 ; OPTM1: je
 
+define internal i32 @shr64BitUnsignedTrunc(i64 %a, i64 %b) {
+entry:
+  %shr = lshr i64 %a, %b
+  %result = trunc i64 %shr to i32
+  ret i32 %result
+}
+; CHECK-LABEL: shr64BitUnsignedTrunc
+; CHECK: shrd
+; CHECK: shr
+; CHECK: test {{.*}}, 32
+; CHECK: je
+;
+; OPTM1-LABEL: shr64BitUnsignedTrunc
+; OPTM1: shrd
+; OPTM1: shr
+; OPTM1: test {{.*}}, 32
+; OPTM1: je
+
 define internal i64 @and64BitSigned(i64 %a, i64 %b) {
 entry:
   %and = and i64 %b, %a