Subzero: Reduce copying of Liveness bitvectors.

src/IceBitVector.h

@@ -308,7 +308,7 @@ public:
   explicit BitVector(unsigned s, bool t = false, Allocator A = Allocator())
       : Size(s), Alloc(std::move(A)) {
     Capacity = NumBitWords(s);
-    Bits = Alloc.allocate(Capacity * sizeof(BitWord));
+    Bits = Alloc.allocate(Capacity);
     init_words(Bits, Capacity, t);
     if (t)
       clear_unused_bits();
@@ -323,7 +323,7 @@ public:
     }
 
     Capacity = NumBitWords(RHS.size());
-    Bits = Alloc.allocate(Capacity * sizeof(BitWord));
+    Bits = Alloc.allocate(Capacity);
     std::memcpy(Bits, RHS.Bits, Capacity * sizeof(BitWord));
   }
 
@@ -335,7 +335,7 @@ public:
 
   ~BitVector() {
     if (Bits != nullptr) {
-      Alloc.deallocate(Bits, Capacity * sizeof(BitWord));
+      Alloc.deallocate(Bits, Capacity);
     }
   }
 
@@ -655,15 +655,21 @@ public:
       return *this;
     }
 
+    // Currently, BitVector is only used by liveness analysis.  With the
+    // following assert, we make sure BitVectors grow in a single step from 0 to
+    // their final capacity, rather than growing slowly and "leaking" memory in
+    // the process.
+    assert(Capacity == 0);
+
     // Grow the bitvector to have enough elements.
     const auto OldCapacity = Capacity;
     Capacity = RHSWords;
     assert(Capacity > 0 && "negative capacity?");
-    BitWord *NewBits = Alloc.allocate(Capacity * sizeof(BitWord));
+    BitWord *NewBits = Alloc.allocate(Capacity);
     std::memcpy(NewBits, RHS.Bits, Capacity * sizeof(BitWord));
 
     // Destroy the old bits.
-    Alloc.deallocate(Bits, OldCapacity * sizeof(BitWord));
+    Alloc.deallocate(Bits, OldCapacity);
     Bits = NewBits;
 
     return *this;
@@ -673,7 +679,7 @@ public:
     if (this == &RHS)
       return *this;
 
-    Alloc.deallocate(Bits, Capacity * sizeof(BitWord));
+    Alloc.deallocate(Bits, Capacity);
     Bits = RHS.Bits;
     Size = RHS.Size;
     Capacity = RHS.Capacity;
@@ -755,9 +761,9 @@ private:
     const auto OldCapacity = Capacity;
     Capacity = std::max(NumBitWords(NewSize), Capacity * 2);
     assert(Capacity > 0 && "realloc-ing zero space");
-    auto *NewBits = Alloc.allocate(Capacity * sizeof(BitWord));
+    auto *NewBits = Alloc.allocate(Capacity);
     std::memcpy(Bits, NewBits, OldCapacity * sizeof(BitWord));
-    Alloc.deallocate(Bits, OldCapacity * sizeof(BitWord));
+    Alloc.deallocate(Bits, OldCapacity);
     Bits = NewBits;
 
     clear_unused_bits();

src/IceCfg.cpp

@@ -1076,6 +1076,14 @@ void Cfg::emitIAS() {
   emitJumpTables();
 }
 
+size_t Cfg::getTotalMemoryMB() {
+  constexpr size_t OneMB = 1024 * 1024;
+  using ArbitraryType = int;
+  // CfgLocalAllocator draws from the same memory pool regardless of allocated
+  // object type, so pick an arbitrary type for the template parameter.
+  return CfgLocalAllocator<ArbitraryType>().current()->getTotalMemory() / OneMB;
+}
+
 // Dumps the IR with an optional introductory message.
 void Cfg::dump(const IceString &Message) {
   if (!BuildDefs::dump())
@@ -1087,10 +1095,7 @@ void Cfg::dump(const IceString &Message) {
   if (!Message.empty())
     Str << "================ " << Message << " ================\n";
   if (isVerbose(IceV_Mem)) {
-    constexpr size_t OneMB = 1024 * 1024;
-    Str << "Memory size = "
-        << (CfgLocalAllocator<int>().current()->getTotalMemory() / OneMB)
-        << " MB\n";
+    Str << "Memory size = " << getTotalMemoryMB() << " MB\n";
   }
   setCurrentNode(getEntryNode());
   // Print function name+args

src/IceCfg.h

@@ -219,6 +219,10 @@ public:
   const CfgNode *getCurrentNode() const { return CurrentNode; }
   /// @}
 
+  /// Get the total amount of memory held by the per-Cfg allocator.  This is
+  /// mostly meant for use inside a debugger.
+  static size_t getTotalMemoryMB();
+
   void emit();
   void emitIAS();
   static void emitTextHeader(const IceString &MangledName, GlobalContext *Ctx,

src/IceCfgNode.cpp

@@ -626,8 +626,11 @@ void CfgNode::livenessLightweight() {
 // liveness changed from before, false if it stayed the same. (If it changes,
 // the node's predecessors need to be processed again.)
 bool CfgNode::liveness(Liveness *Liveness) {
-  SizeT NumVars = Liveness->getNumVarsInNode(this);
-  LivenessBV Live(NumVars);
+  const SizeT NumVars = Liveness->getNumVarsInNode(this);
+  const SizeT NumGlobalVars = Liveness->getNumGlobalVars();
+  LivenessBV &Live = Liveness->getScratchBV();
+  Live.clear();
+
   LiveBeginEndMap *LiveBegin = nullptr;
   LiveBeginEndMap *LiveEnd = nullptr;
   // Mark the beginning and ending of each variable's live range with the
@@ -642,19 +645,25 @@ bool CfgNode::liveness(Liveness *Liveness) {
     LiveBegin->reserve(getInstCountEstimate());
     LiveEnd->reserve(getInstCountEstimate());
   }
+
   // Initialize Live to be the union of all successors' LiveIn.
   for (CfgNode *Succ : OutEdges) {
-    Live |= Liveness->getLiveIn(Succ);
+    const LivenessBV &LiveIn = Liveness->getLiveIn(Succ);
+    assert(LiveIn.empty() || LiveIn.size() == NumGlobalVars);
+    Live |= LiveIn;
     // Mark corresponding argument of phis in successor as live.
     for (Inst &I : Succ->Phis) {
       if (I.isDeleted())
         continue;
-      auto *Phi = llvm::dyn_cast<InstPhi>(&I);
+      auto *Phi = llvm::cast<InstPhi>(&I);
       Phi->livenessPhiOperand(Live, this, Liveness);
     }
   }
+  assert(Live.empty() || Live.size() == NumGlobalVars);
   Liveness->getLiveOut(this) = Live;
 
+  // Expand Live so it can hold locals in addition to globals.
+  Live.resize(NumVars);
   // Process regular instructions in reverse order.
   for (Inst &I : reverse_range(Insts)) {
     if (I.isDeleted())
@@ -676,20 +685,17 @@ bool CfgNode::liveness(Liveness *Liveness) {
 
   // When using the sparse representation, after traversing the instructions in
   // the block, the Live bitvector should only contain set bits for global
-  // variables upon block entry. We validate this by shrinking the Live vector
-  // and then testing it against the pre-shrunk version. (The shrinking is
-  // required, but the validation is not.)
-  LivenessBV LiveOrig = Live;
-  Live.resize(Liveness->getNumGlobalVars());
-  if (Live != LiveOrig) {
+  // variables upon block entry.  We validate this by testing the upper bits of
+  // the Live bitvector.
+  if (Live.find_next(NumGlobalVars) != -1) {
     if (BuildDefs::dump()) {
       // This is a fatal liveness consistency error. Print some diagnostics and
       // abort.
       Ostream &Str = Func->getContext()->getStrDump();
       Func->resetCurrentNode();
-      Str << "LiveOrig-Live =";
-      for (SizeT i = Live.size(); i < LiveOrig.size(); ++i) {
-        if (LiveOrig.test(i)) {
+      Str << "Invalid Live =";
+      for (SizeT i = NumGlobalVars; i < Live.size(); ++i) {
+        if (Live.test(i)) {
           Str << " ";
           Liveness->getVariable(i, this)->dump(Func);
         }
@@ -698,9 +704,12 @@ bool CfgNode::liveness(Liveness *Liveness) {
     }
     llvm::report_fatal_error("Fatal inconsistency in liveness analysis");
   }
+  // Now truncate Live to prevent LiveIn from growing.
+  Live.resize(NumGlobalVars);
 
   bool Changed = false;
   LivenessBV &LiveIn = Liveness->getLiveIn(this);
+  assert(LiveIn.empty() || LiveIn.size() == NumGlobalVars);
   // Add in current LiveIn
   Live |= LiveIn;
   // Check result, set LiveIn=Live
@@ -778,9 +787,9 @@ void CfgNode::livenessAddIntervals(Liveness *Liveness, InstNumberT FirstInstNum,
                                    InstNumberT LastInstNum) {
   TimerMarker T1(TimerStack::TT_liveRange, Func);
 
-  SizeT NumVars = Liveness->getNumVarsInNode(this);
-  LivenessBV &LiveIn = Liveness->getLiveIn(this);
-  LivenessBV &LiveOut = Liveness->getLiveOut(this);
+  const SizeT NumVars = Liveness->getNumVarsInNode(this);
+  const LivenessBV &LiveIn = Liveness->getLiveIn(this);
+  const LivenessBV &LiveOut = Liveness->getLiveOut(this);
   LiveBeginEndMap &MapBegin = *Liveness->getLiveBegin(this);
   LiveBeginEndMap &MapEnd = *Liveness->getLiveEnd(this);
   std::sort(MapBegin.begin(), MapBegin.end());
@@ -791,7 +800,8 @@ void CfgNode::livenessAddIntervals(Liveness *Liveness, InstNumberT FirstInstNum,
     return;
   }
 
-  LivenessBV LiveInAndOut = LiveIn;
+  LivenessBV &LiveInAndOut = Liveness->getScratchBV();
+  LiveInAndOut = LiveIn;
   LiveInAndOut &= LiveOut;
 
   // Iterate in parallel across the sorted MapBegin[] and MapEnd[].
@@ -1350,23 +1360,23 @@ void CfgNode::dump(Cfg *Func) const {
     Str << "\n";
   }
   // Dump the live-in variables.
-  LivenessBV LiveIn;
-  if (Liveness)
-    LiveIn = Liveness->getLiveIn(this);
-  if (Func->isVerbose(IceV_Liveness) && !LiveIn.empty()) {
-    Str << "    // LiveIn:";
-    for (SizeT i = 0; i < LiveIn.size(); ++i) {
-      if (LiveIn[i]) {
-        Variable *Var = Liveness->getVariable(i, this);
-        Str << " %" << Var->getName(Func);
-        if (Func->isVerbose(IceV_RegOrigins) && Var->hasReg()) {
-          Str << ":"
-              << Func->getTarget()->getRegName(Var->getRegNum(),
-                                               Var->getType());
+  if (Func->isVerbose(IceV_Liveness)) {
+    if (Liveness != nullptr && !Liveness->getLiveIn(this).empty()) {
+      const LivenessBV &LiveIn = Liveness->getLiveIn(this);
+      Str << "    // LiveIn:";
+      for (SizeT i = 0; i < LiveIn.size(); ++i) {
+        if (LiveIn[i]) {
+          Variable *Var = Liveness->getVariable(i, this);
+          Str << " %" << Var->getName(Func);
+          if (Func->isVerbose(IceV_RegOrigins) && Var->hasReg()) {
+            Str << ":"
+                << Func->getTarget()->getRegName(Var->getRegNum(),
+                                                 Var->getType());
+          }
         }
       }
+      Str << "\n";
     }
-    Str << "\n";
   }
   // Dump each instruction.
   if (Func->isVerbose(IceV_Instructions)) {
@@ -1376,23 +1386,23 @@ void CfgNode::dump(Cfg *Func) const {
       I.dumpDecorated(Func);
   }
   // Dump the live-out variables.
-  LivenessBV LiveOut;
-  if (Liveness)
-    LiveOut = Liveness->getLiveOut(this);
-  if (Func->isVerbose(IceV_Liveness) && !LiveOut.empty()) {
-    Str << "    // LiveOut:";
-    for (SizeT i = 0; i < LiveOut.size(); ++i) {
-      if (LiveOut[i]) {
-        Variable *Var = Liveness->getVariable(i, this);
-        Str << " %" << Var->getName(Func);
-        if (Func->isVerbose(IceV_RegOrigins) && Var->hasReg()) {
-          Str << ":"
-              << Func->getTarget()->getRegName(Var->getRegNum(),
-                                               Var->getType());
+  if (Func->isVerbose(IceV_Liveness)) {
+    if (Liveness != nullptr && !Liveness->getLiveOut(this).empty()) {
+      const LivenessBV &LiveOut = Liveness->getLiveOut(this);
+      Str << "    // LiveOut:";
+      for (SizeT i = 0; i < LiveOut.size(); ++i) {
+        if (LiveOut[i]) {
+          Variable *Var = Liveness->getVariable(i, this);
+          Str << " %" << Var->getName(Func);
+          if (Func->isVerbose(IceV_RegOrigins) && Var->hasReg()) {
+            Str << ":"
+                << Func->getTarget()->getRegName(Var->getRegNum(),
+                                                 Var->getType());
+          }
         }
       }
+      Str << "\n";
     }
-    Str << "\n";
   }
   // Dump list of successor nodes.
   if (Func->isVerbose(IceV_Succs)) {

src/IceInstX86BaseImpl.h

@@ -98,8 +98,11 @@ InstImpl<TraitsType>::InstX86Label::InstX86Label(Cfg *Func,
 
 template <typename TraitsType>
 IceString InstImpl<TraitsType>::InstX86Label::getName(const Cfg *Func) const {
+  // TODO(stichnot): Returning an empty string in a non-DUMP build can cause a
+  // huge degradation in ConstantRelocatable hashing.  Investigate and fix, but
+  // for now return something reasonably unique.
   if (!BuildDefs::dump())
-    return IceString();
+    return Func->getFunctionName() + std::to_string(Number);
   return ".L" + Func->getFunctionName() + "$local$__" + std::to_string(Number);
 }
 

src/IceLiveness.cpp

@@ -106,6 +106,7 @@ void Liveness::initInternal(NodeList::const_iterator FirstNode,
       RangeMask[VarIndex] = false;
   }
 
+  SizeT MaxLocals = 0;
   // Process each node.
   for (auto I = FirstNode, E = Func->getNodes().end(); I != E; ++I) {
     LivenessNode &Node = Nodes[(*I)->getIndex()];
@@ -113,7 +114,9 @@ void Liveness::initInternal(NodeList::const_iterator FirstNode,
     Node.LiveIn.resize(NumGlobals);
     Node.LiveOut.resize(NumGlobals);
     // LiveBegin and LiveEnd are reinitialized before each pass over the block.
+    MaxLocals = std::max(MaxLocals, Node.NumLocals);
   }
+  ScratchBV.reserve(NumGlobals + MaxLocals);
 }
 
 void Liveness::init() {

src/IceLiveness.h

@@ -85,6 +85,7 @@ public:
     resize(Index);
     return Nodes[Index].LiveOut;
   }
+  LivenessBV &getScratchBV() { return ScratchBV; }
   LiveBeginEndMap *getLiveBegin(const CfgNode *Node) {
     SizeT Index = Node->getIndex();
     resize(Index);
@@ -119,6 +120,9 @@ private:
   /// RangeMask[Variable::Number] indicates whether we want to track that
   /// Variable's live range.
   LivenessBV RangeMask;
+  /// ScratchBV is a bitvector that can be reused across CfgNode passes, to
+  /// avoid having to allocate/deallocate memory so frequently.
+  LivenessBV ScratchBV;
 };
 
 } // end of namespace Ice