Subzero. Liveness memory management.

src/IceBitVector.h

@@ -246,9 +246,9 @@ private:
   }
 };
 
-class BitVector {
+template <template <typename> class AT> class BitVectorTmpl {
   typedef unsigned long BitWord;
-  using Allocator = CfgLocalAllocator<BitWord>;
+  using Allocator = AT<BitWord>;
 
   enum { BITWORD_SIZE = (unsigned)sizeof(BitWord) * CHAR_BIT };
 
@@ -264,7 +264,7 @@ public:
   typedef unsigned size_type;
   // Encapsulation of a single bit.
   class reference {
-    friend class BitVector;
+    friend class BitVectorTmpl;
 
     BitWord *WordRef;
     unsigned BitPos;
@@ -272,7 +272,7 @@ public:
     reference(); // Undefined
 
   public:
-    reference(BitVector &b, unsigned Idx) {
+    reference(BitVectorTmpl &b, unsigned Idx) {
       WordRef = &b.Bits[Idx / BITWORD_SIZE];
       BitPos = Idx % BITWORD_SIZE;
     }
@@ -297,15 +297,15 @@ public:
     }
   };
 
-  /// BitVector default ctor - Creates an empty bitvector.
-  BitVector(Allocator A = Allocator())
+  /// BitVectorTmpl default ctor - Creates an empty bitvector.
+  BitVectorTmpl(Allocator A = Allocator())
       : Size(0), Capacity(0), Alloc(std::move(A)) {
     Bits = nullptr;
   }
 
-  /// BitVector ctor - Creates a bitvector of specified number of bits. All
+  /// BitVectorTmpl ctor - Creates a bitvector of specified number of bits. All
   /// bits are initialized to the specified value.
-  explicit BitVector(unsigned s, bool t = false, Allocator A = Allocator())
+  explicit BitVectorTmpl(unsigned s, bool t = false, Allocator A = Allocator())
       : Size(s), Alloc(std::move(A)) {
     Capacity = NumBitWords(s);
     Bits = Alloc.allocate(Capacity);
@@ -314,8 +314,8 @@ public:
       clear_unused_bits();
   }
 
-  /// BitVector copy ctor.
-  BitVector(const BitVector &RHS) : Size(RHS.size()), Alloc(RHS.Alloc) {
+  /// BitVectorTmpl copy ctor.
+  BitVectorTmpl(const BitVectorTmpl &RHS) : Size(RHS.size()), Alloc(RHS.Alloc) {
     if (Size == 0) {
       Bits = nullptr;
       Capacity = 0;
@@ -327,13 +327,13 @@ public:
     std::memcpy(Bits, RHS.Bits, Capacity * sizeof(BitWord));
   }
 
-  BitVector(BitVector &&RHS)
+  BitVectorTmpl(BitVectorTmpl &&RHS)
       : Bits(RHS.Bits), Size(RHS.Size), Capacity(RHS.Capacity),
         Alloc(std::move(RHS.Alloc)) {
     RHS.Bits = nullptr;
   }
 
-  ~BitVector() {
+  ~BitVectorTmpl() {
     if (Bits != nullptr) {
       Alloc.deallocate(Bits, Capacity);
     }
@@ -420,7 +420,7 @@ public:
       init_words(&Bits[OldCapacity], (Capacity - OldCapacity), t);
     }
 
-    // Set any old unused bits that are now included in the BitVector. This
+    // Set any old unused bits that are now included in the BitVectorTmpl. This
     // may set bits that are not included in the new vector, but we will clear
     // them back out below.
     if (N > Size)
@@ -439,20 +439,20 @@ public:
   }
 
   // Set, reset, flip
-  BitVector &set() {
+  BitVectorTmpl &set() {
     init_words(Bits, Capacity, true);
     clear_unused_bits();
     return *this;
   }
 
-  BitVector &set(unsigned Idx) {
+  BitVectorTmpl &set(unsigned Idx) {
     assert(Bits && "Bits never allocated");
     Bits[Idx / BITWORD_SIZE] |= BitWord(1) << (Idx % BITWORD_SIZE);
     return *this;
   }
 
   /// set - Efficiently set a range of bits in [I, E)
-  BitVector &set(unsigned I, unsigned E) {
+  BitVectorTmpl &set(unsigned I, unsigned E) {
     assert(I <= E && "Attempted to set backwards range!");
     assert(E <= size() && "Attempted to set out-of-bounds range!");
 
@@ -481,18 +481,18 @@ public:
     return *this;
   }
 
-  BitVector &reset() {
+  BitVectorTmpl &reset() {
     init_words(Bits, Capacity, false);
     return *this;
   }
 
-  BitVector &reset(unsigned Idx) {
+  BitVectorTmpl &reset(unsigned Idx) {
     Bits[Idx / BITWORD_SIZE] &= ~(BitWord(1) << (Idx % BITWORD_SIZE));
     return *this;
   }
 
   /// reset - Efficiently reset a range of bits in [I, E)
-  BitVector &reset(unsigned I, unsigned E) {
+  BitVectorTmpl &reset(unsigned I, unsigned E) {
     assert(I <= E && "Attempted to reset backwards range!");
     assert(E <= size() && "Attempted to reset out-of-bounds range!");
 
@@ -521,14 +521,14 @@ public:
     return *this;
   }
 
-  BitVector &flip() {
+  BitVectorTmpl &flip() {
     for (unsigned i = 0; i < NumBitWords(size()); ++i)
       Bits[i] = ~Bits[i];
     clear_unused_bits();
     return *this;
   }
 
-  BitVector &flip(unsigned Idx) {
+  BitVectorTmpl &flip(unsigned Idx) {
     Bits[Idx / BITWORD_SIZE] ^= BitWord(1) << (Idx % BITWORD_SIZE);
     return *this;
   }
@@ -548,7 +548,7 @@ public:
   bool test(unsigned Idx) const { return (*this)[Idx]; }
 
   /// Test if any common bits are set.
-  bool anyCommon(const BitVector &RHS) const {
+  bool anyCommon(const BitVectorTmpl &RHS) const {
     unsigned ThisWords = NumBitWords(size());
     unsigned RHSWords = NumBitWords(RHS.size());
     for (unsigned i = 0, e = std::min(ThisWords, RHSWords); i != e; ++i)
@@ -558,7 +558,7 @@ public:
   }
 
   // Comparison operators.
-  bool operator==(const BitVector &RHS) const {
+  bool operator==(const BitVectorTmpl &RHS) const {
     unsigned ThisWords = NumBitWords(size());
     unsigned RHSWords = NumBitWords(RHS.size());
     unsigned i;
@@ -579,10 +579,10 @@ public:
     return true;
   }
 
-  bool operator!=(const BitVector &RHS) const { return !(*this == RHS); }
+  bool operator!=(const BitVectorTmpl &RHS) const { return !(*this == RHS); }
 
   /// Intersection, union, disjoint union.
-  BitVector &operator&=(const BitVector &RHS) {
+  BitVectorTmpl &operator&=(const BitVectorTmpl &RHS) {
     unsigned ThisWords = NumBitWords(size());
     unsigned RHSWords = NumBitWords(RHS.size());
     unsigned i;
@@ -599,7 +599,7 @@ public:
   }
 
   /// reset - Reset bits that are set in RHS. Same as *this &= ~RHS.
-  BitVector &reset(const BitVector &RHS) {
+  BitVectorTmpl &reset(const BitVectorTmpl &RHS) {
     unsigned ThisWords = NumBitWords(size());
     unsigned RHSWords = NumBitWords(RHS.size());
     unsigned i;
@@ -610,7 +610,7 @@ public:
 
   /// test - Check if (This - RHS) is zero.
   /// This is the same as reset(RHS) and any().
-  bool test(const BitVector &RHS) const {
+  bool test(const BitVectorTmpl &RHS) const {
     unsigned ThisWords = NumBitWords(size());
     unsigned RHSWords = NumBitWords(RHS.size());
     unsigned i;
@@ -625,7 +625,7 @@ public:
     return false;
   }
 
-  BitVector &operator|=(const BitVector &RHS) {
+  BitVectorTmpl &operator|=(const BitVectorTmpl &RHS) {
     if (size() < RHS.size())
       resize(RHS.size());
     for (size_t i = 0, e = NumBitWords(RHS.size()); i != e; ++i)
@@ -633,7 +633,7 @@ public:
     return *this;
   }
 
-  BitVector &operator^=(const BitVector &RHS) {
+  BitVectorTmpl &operator^=(const BitVectorTmpl &RHS) {
     if (size() < RHS.size())
       resize(RHS.size());
     for (size_t i = 0, e = NumBitWords(RHS.size()); i != e; ++i)
@@ -642,7 +642,7 @@ public:
   }
 
   // Assignment operator.
-  const BitVector &operator=(const BitVector &RHS) {
+  const BitVectorTmpl &operator=(const BitVectorTmpl &RHS) {
     if (this == &RHS)
       return *this;
 
@@ -655,10 +655,10 @@ public:
       return *this;
     }
 
-    // Currently, BitVector is only used by liveness analysis.  With the
-    // following assert, we make sure BitVectors grow in a single step from 0 to
-    // their final capacity, rather than growing slowly and "leaking" memory in
-    // the process.
+    // Currently, BitVectorTmpl is only used by liveness analysis.  With the
+    // following assert, we make sure BitVectorTmpls grow in a single step from
+    // 0 to their final capacity, rather than growing slowly and "leaking"
+    // memory in the process.
     assert(Capacity == 0);
 
     // Grow the bitvector to have enough elements.
@@ -675,7 +675,7 @@ public:
     return *this;
   }
 
-  const BitVector &operator=(BitVector &&RHS) {
+  const BitVectorTmpl &operator=(BitVectorTmpl &&RHS) {
     if (this == &RHS)
       return *this;
 
@@ -689,7 +689,7 @@ public:
     return *this;
   }
 
-  void swap(BitVector &RHS) {
+  void swap(BitVectorTmpl &RHS) {
     std::swap(Bits, RHS.Bits);
     std::swap(Size, RHS.Size);
     std::swap(Capacity, RHS.Capacity);
@@ -705,7 +705,7 @@ public:
   //
   // The LSB in each word is the lowest numbered bit.  The size of a portable
   // bit mask is always a whole multiple of 32 bits.  If no bit mask size is
-  // given, the bit mask is assumed to cover the entire BitVector.
+  // given, the bit mask is assumed to cover the entire BitVectorTmpl.
 
   /// setBitsInMask - Add '1' bits from Mask to this vector. Don't resize.
   /// This computes "*this |= Mask".
@@ -807,11 +807,16 @@ private:
   }
 };
 
+using BitVector = BitVectorTmpl<CfgLocalAllocator>;
+
 } // end of namespace Ice
 
 namespace std {
-/// Implement std::swap in terms of BitVector swap.
-inline void swap(Ice::BitVector &LHS, Ice::BitVector &RHS) { LHS.swap(RHS); }
+/// Implement std::swap in terms of BitVectorTmpl swap.
+template <template <typename> class AT>
+inline void swap(Ice::BitVectorTmpl<AT> &LHS, Ice::BitVectorTmpl<AT> &RHS) {
+  LHS.swap(RHS);
+}
 }
 
 #endif // SUBZERO_SRC_ICEBITVECTOR_H

src/IceCfg.cpp

@@ -28,6 +28,9 @@
 #include "IceOperand.h"
 #include "IceTargetLowering.h"
 
+#include <memory>
+#include <utility>
+
 namespace Ice {
 
 Cfg::Cfg(GlobalContext *Ctx, uint32_t SequenceNumber)
@@ -819,9 +822,14 @@ void Cfg::livenessLightweight() {
 
 void Cfg::liveness(LivenessMode Mode) {
   TimerMarker T(TimerStack::TT_liveness, this);
-  Live.reset(new Liveness(this, Mode));
+  // Destroying the previous (if any) Liveness information clears the Liveness
+  // allocator TLS pointer.
+  Live = nullptr;
+  Live = Liveness::create(this, Mode);
+
   getVMetadata()->init(VMK_Uses);
   Live->init();
+
   // Initialize with all nodes needing to be processed.
   BitVector NeedToProcess(Nodes.size(), true);
   while (NeedToProcess.any()) {

src/IceDefs.h

@@ -49,7 +49,7 @@
 namespace Ice {
 
 class Assembler;
-class BitVector;
+template <template <typename> class> class BitVectorTmpl;
 class Cfg;
 class CfgNode;
 class Constant;
@@ -272,9 +272,11 @@ using InstNumberT = int32_t;
 /// A LiveBeginEndMapEntry maps a Variable::Number value to an Inst::Number
 /// value, giving the instruction number that begins or ends a variable's live
 /// range.
+template <typename T>
+using LivenessVector = std::vector<T, LivenessAllocator<T>>;
 using LiveBeginEndMapEntry = std::pair<SizeT, InstNumberT>;
-using LiveBeginEndMap = CfgVector<LiveBeginEndMapEntry>;
-using LivenessBV = BitVector;
+using LiveBeginEndMap = LivenessVector<LiveBeginEndMapEntry>;
+using LivenessBV = BitVectorTmpl<LivenessAllocator>;
 
 using TimerStackIdT = uint32_t;
 using TimerIdT = uint32_t;

src/IceGlobalContext.cpp

@@ -21,6 +21,7 @@
 #include "IceDefs.h"
 #include "IceELFObjectWriter.h"
 #include "IceGlobalInits.h"
+#include "IceLiveness.h"
 #include "IceOperand.h"
 #include "IceTargetLowering.h"
 #include "IceTimerTree.h"
@@ -304,6 +305,7 @@ GlobalContext::GlobalContext(Ostream *OsDump, Ostream *OsEmit, Ostream *OsError,
   // free.
   GlobalContext::TlsInit();
   Cfg::TlsInit();
+  Liveness::TlsInit();
   // Create a new ThreadContext for the current thread.  No need to
   // lock AllThreadContexts at this point since no other threads have
   // access yet to this GlobalContext object.

src/IceLiveness.cpp

@@ -106,8 +106,8 @@ void Liveness::initInternal(NodeList::const_iterator FirstNode,
       RangeMask[VarIndex] = false;
   }
 
-  SizeT MaxLocals = 0;
   // Process each node.
+  MaxLocals = 0;
   for (auto I = FirstNode, E = Func->getNodes().end(); I != E; ++I) {
     LivenessNode &Node = Nodes[(*I)->getIndex()];
     // NumLocals, LiveToVarMap already initialized

src/IceLiveness.h

@@ -24,8 +24,12 @@
 #include "IceDefs.h"
 #include "IceBitVector.h"
 #include "IceCfgNode.h"
+#include "IceTLS.h"
 #include "IceTypes.h"
 
+#include <memory>
+#include <utility>
+
 namespace Ice {
 
 class Liveness {
@@ -49,7 +53,7 @@ class Liveness {
     // LiveToVarMap maps a liveness bitvector index to a Variable. This is
     // generally just for printing/dumping. The index should be less than
     // NumLocals + Liveness::NumGlobals.
-    CfgVector<Variable *> LiveToVarMap;
+    LivenessVector<Variable *> LiveToVarMap;
     // LiveIn and LiveOut track the in- and out-liveness of the global
     // variables. The size of each vector is LivenessNode::NumGlobals.
     LivenessBV LiveIn, LiveOut;
@@ -60,7 +64,6 @@ class Liveness {
   };
 
 public:
-  Liveness(Cfg *Func, LivenessMode Mode) : Func(Func), Mode(Mode) {}
   void init();
   void initPhiEdgeSplits(NodeList::const_iterator FirstNode,
                          VarList::const_iterator FirstVar);
@@ -102,32 +105,58 @@ public:
   }
   bool getRangeMask(SizeT Index) const { return RangeMask[Index]; }
 
+  ArenaAllocator *getAllocator() const { return Alloc.get(); }
+
+  static std::unique_ptr<Liveness> create(Cfg *Func, LivenessMode Mode) {
+    return std::unique_ptr<Liveness>(new Liveness(Func, Mode));
+  }
+
+  static void TlsInit() { LivenessAllocatorTraits::init(); }
+
+  std::string dumpStr() const {
+    return "MaxLocals(" + std::to_string(MaxLocals) + "), "
+                                                      "NumGlobals(" +
+           std::to_string(NumGlobals) + ")";
+  }
+
 private:
+  Liveness(Cfg *Func, LivenessMode Mode)
+      : Alloc(new ArenaAllocator()), AllocScope(this), Func(Func), Mode(Mode) {}
+
   void initInternal(NodeList::const_iterator FirstNode,
                     VarList::const_iterator FirstVar, bool IsFullInit);
   /// Resize Nodes so that Nodes[Index] is valid.
   void resize(SizeT Index) {
-    if (Index >= Nodes.size())
+    if (Index >= Nodes.size()) {
+      assert(false && "The Nodes array is not expected to be resized.");
       Nodes.resize(Index + 1);
+    }
   }
+  std::unique_ptr<ArenaAllocator> Alloc;
+  LivenessAllocatorScope AllocScope; // Must be declared after Alloc.
   static constexpr SizeT InvalidLiveIndex = -1;
   Cfg *Func;
   LivenessMode Mode;
-  SizeT NumGlobals = 0;
   /// Size of Nodes is Cfg::Nodes.size().
-  CfgVector<LivenessNode> Nodes;
+  LivenessVector<LivenessNode> Nodes;
   /// VarToLiveMap maps a Variable's Variable::Number to its live index within
   /// its basic block.
-  CfgVector<SizeT> VarToLiveMap;
+  LivenessVector<SizeT> VarToLiveMap;
   /// LiveToVarMap is analogous to LivenessNode::LiveToVarMap, but for non-local
   /// variables.
-  CfgVector<Variable *> LiveToVarMap;
+  LivenessVector<Variable *> LiveToVarMap;
   /// RangeMask[Variable::Number] indicates whether we want to track that
   /// Variable's live range.
   LivenessBV RangeMask;
   /// ScratchBV is a bitvector that can be reused across CfgNode passes, to
   /// avoid having to allocate/deallocate memory so frequently.
   LivenessBV ScratchBV;
+  /// MaxLocals indicates what is the maximum number of local variables in a
+  /// single basic block, across all blocks in a function.
+  SizeT MaxLocals = 0;
+  /// NumGlobals indicates how many global variables (i.e., Multi Block) exist
+  /// for a function.
+  SizeT NumGlobals = 0;
 };
 
 } // end of namespace Ice

src/IceMemory.cpp

@@ -15,6 +15,7 @@
 #include "IceMemory.h"
 
 #include "IceCfg.h"
+#include "IceLiveness.h"
 #include "IceTLS.h"
 
 #include <cassert>
@@ -33,4 +34,16 @@ void CfgAllocatorTraits::set_current(const manager_type *Manager) {
   ICE_TLS_SET_FIELD(CfgAllocator, Allocator);
 }
 
+ICE_TLS_DEFINE_FIELD(ArenaAllocator *, LivenessAllocatorTraits,
+                     LivenessAllocator);
+
+LivenessAllocatorTraits::allocator_type LivenessAllocatorTraits::current() {
+  return ICE_TLS_GET_FIELD(LivenessAllocator);
+}
+
+void LivenessAllocatorTraits::set_current(const manager_type *Manager) {
+  ArenaAllocator *Allocator =
+      Manager == nullptr ? nullptr : Manager->getAllocator();
+  ICE_TLS_SET_FIELD(LivenessAllocator, Allocator);
+}
 } // end of namespace Ice

src/IceMemory.h

@@ -27,6 +27,7 @@ namespace Ice {
 
 class Cfg;
 class GlobalContext;
+class Liveness;
 
 using ArenaAllocator =
     llvm::BumpPtrAllocatorImpl<llvm::MallocAllocator, /*SlabSize=*/1024 * 1024>;
@@ -61,8 +62,10 @@ template <typename T, typename Traits> struct sz_allocator {
   using size_type = std::size_t;
   using difference_type = std::ptrdiff_t;
 
-  sz_allocator() = default;
-  template <class U> sz_allocator(const sz_allocator<U, Traits> &) {}
+  sz_allocator() : Current() {}
+  template <class U>
+  sz_allocator(const sz_allocator<U, Traits> &)
+      : Current() {}
 
   pointer address(reference x) const {
     return reinterpret_cast<pointer>(&reinterpret_cast<char &>(x));
@@ -72,6 +75,7 @@ template <typename T, typename Traits> struct sz_allocator {
   }
 
   pointer allocate(size_type num) {
+    assert(current() != nullptr);
     return current()->template Allocate<T>(num);
   }
 
@@ -88,9 +92,20 @@ template <typename T, typename Traits> struct sz_allocator {
 
   /// Manages the current underlying allocator.
   /// @{
-  static typename Traits::allocator_type current() { return Traits::current(); }
+  typename Traits::allocator_type current() {
+    if (!Traits::cache_allocator) {
+      // TODO(jpp): allocators should always be cacheable... maybe. Investigate.
+      return Traits::current();
+    }
+    if (Current == nullptr) {
+      Current = Traits::current();
+    }
+    assert(Current == Traits::current());
+    return Current;
+  }
   static void init() { Traits::init(); }
   /// @}
+  typename Traits::allocator_type Current;
 };
 
 template <class Traits> struct sz_allocator_scope {
@@ -122,6 +137,7 @@ class CfgAllocatorTraits {
 public:
   using allocator_type = ArenaAllocator *;
   using manager_type = Cfg;
+  static constexpr bool cache_allocator = false;
 
   static void init() { ICE_TLS_INIT_FIELD(CfgAllocator); };
 
@@ -137,6 +153,31 @@ using CfgLocalAllocator = sz_allocator<T, CfgAllocatorTraits>;
 
 using CfgLocalAllocatorScope = sz_allocator_scope<CfgAllocatorTraits>;
 
+class LivenessAllocatorTraits {
+  LivenessAllocatorTraits() = delete;
+  LivenessAllocatorTraits(const LivenessAllocatorTraits &) = delete;
+  LivenessAllocatorTraits &operator=(const LivenessAllocatorTraits &) = delete;
+  ~LivenessAllocatorTraits() = delete;
+
+public:
+  using allocator_type = ArenaAllocator *;
+  using manager_type = Liveness;
+  static constexpr bool cache_allocator = true;
+
+  static void init() { ICE_TLS_INIT_FIELD(LivenessAllocator); };
+
+  static allocator_type current();
+  static void set_current(const manager_type *Manager);
+
+private:
+  ICE_TLS_DECLARE_FIELD(ArenaAllocator *, LivenessAllocator);
+};
+
+template <typename T>
+using LivenessAllocator = sz_allocator<T, LivenessAllocatorTraits>;
+
+using LivenessAllocatorScope = sz_allocator_scope<LivenessAllocatorTraits>;
+
 } // end of namespace Ice
 
 #endif // SUBZERO_SRC_ICEMEMORY_H