Subzero: Implementation of "advanced Phi lowering".

pydir/szbuild_spec2k.py

@@ -16,10 +16,11 @@ def main():
     './run_all.sh RunBenchmarks SetupGccX8632Opt {train|ref} ...'
     """
     nacl_root = FindBaseNaCl()
-    components = [ '164.gzip', '175.vpr', '176.gcc', '177.mesa', '179.art', 
-                   '181.mcf', '183.equake', '186.crafty', '188.ammp',
-                   '197.parser', '252.eon', '253.perlbmk', '254.gap',
-                   '255.vortex', '256.bzip2', '300.twolf' ]
+    # Use the same default ordering as spec2k/run_all.sh.
+    components = [ '177.mesa', '179.art', '183.equake', '188.ammp', '164.gzip',
+                   '175.vpr', '176.gcc', '181.mcf', '186.crafty', '197.parser',
+                   '253.perlbmk', '254.gap', '255.vortex', '256.bzip2',
+                   '300.twolf', '252.eon' ]
     
     argparser = argparse.ArgumentParser(description=main.__doc__)
     szbuild.AddOptionalArgs(argparser)

src/IceCfg.cpp

@@ -116,6 +116,87 @@ void Cfg::deletePhis() {
     Node->deletePhis();
 }
 
+void Cfg::advancedPhiLowering() {
+  TimerMarker T(TimerStack::TT_advancedPhiLowering, this);
+  // This splits edges and appends new nodes to the end of the node
+  // list.  This can invalidate iterators, so don't use an iterator.
+  SizeT NumNodes = getNumNodes();
+  for (SizeT I = 0; I < NumNodes; ++I)
+    Nodes[I]->advancedPhiLowering();
+}
+
+// Find a reasonable placement for nodes that have not yet been
+// placed, while maintaining the same relative ordering among already
+// placed nodes.
+void Cfg::reorderNodes() {
+  typedef std::list<CfgNode *> PlacedList;
+  PlacedList Placed;      // Nodes with relative placement locked down
+  PlacedList Unreachable; // Unreachable nodes
+  PlacedList::iterator NoPlace = Placed.end();
+  // Keep track of where each node has been tentatively placed so that
+  // we can manage insertions into the middle.
+  std::vector<PlacedList::iterator> PlaceIndex(Nodes.size(), NoPlace);
+  for (CfgNode *Node : Nodes) {
+    // The "do ... while(0);" construct is to factor out the
+    // --PlaceIndex and assert() statements before moving to the next
+    // node.
+    do {
+      if (!Node->needsPlacement()) {
+        // Add to the end of the Placed list.
+        Placed.push_back(Node);
+        PlaceIndex[Node->getIndex()] = Placed.end();
+        continue;
+      }
+      Node->setNeedsPlacement(false);
+      if (Node != getEntryNode() && Node->getInEdges().size() == 0) {
+        // The node has essentially been deleted since it is not a
+        // successor of any other node.
+        Unreachable.push_back(Node);
+        PlaceIndex[Node->getIndex()] = Unreachable.end();
+        continue;
+      }
+      // Assume for now that the unplaced node is from edge-splitting
+      // and therefore has 1 in-edge and 1 out-edge (actually, possibly
+      // more than 1 in-edge if the predecessor node was contracted).
+      // If this changes in the future, rethink the strategy.
+      assert(Node->getInEdges().size() >= 1);
+      assert(Node->getOutEdges().size() == 1);
+
+      // If it's a (non-critical) edge where the successor has a single
+      // in-edge, then place it before the successor.
+      CfgNode *Succ = Node->getOutEdges()[0];
+      if (Succ->getInEdges().size() == 1 &&
+          PlaceIndex[Succ->getIndex()] != NoPlace) {
+        Placed.insert(PlaceIndex[Succ->getIndex()], Node);
+        PlaceIndex[Node->getIndex()] = PlaceIndex[Succ->getIndex()];
+        continue;
+      }
+
+      // Otherwise, place it after the (first) predecessor.
+      CfgNode *Pred = Node->getInEdges()[0];
+      auto PredPosition = PlaceIndex[Pred->getIndex()];
+      // It shouldn't be the case that PredPosition==NoPlace, but if
+      // that somehow turns out to be true, we just insert Node before
+      // PredPosition=NoPlace=Placed.end() .
+      if (PredPosition != NoPlace)
+        ++PredPosition;
+      Placed.insert(PredPosition, Node);
+      PlaceIndex[Node->getIndex()] = PredPosition;
+    } while (0);
+
+    --PlaceIndex[Node->getIndex()];
+    assert(*PlaceIndex[Node->getIndex()] == Node);
+  }
+
+  // Reorder Nodes according to the built-up lists.
+  SizeT Cur = 0;
+  for (CfgNode *Node : Placed)
+    Nodes[Cur++] = Node;
+  for (CfgNode *Node : Unreachable)
+    Nodes[Cur++] = Node;
+  assert(Cur == Nodes.size());
+}
+
 void Cfg::doArgLowering() {
   TimerMarker T(TimerStack::TT_doArgLowering, this);
   getTarget()->lowerArguments();
@@ -297,6 +378,12 @@ void Cfg::deleteRedundantAssignments() {
   }
 }
 
+void Cfg::contractEmptyNodes() {
+  for (CfgNode *Node : Nodes) {
+    Node->contractIfEmpty();
+  }
+}
+
 void Cfg::doBranchOpt() {
   TimerMarker T(TimerStack::TT_doBranchOpt, this);
   for (auto I = Nodes.begin(), E = Nodes.end(); I != E; ++I) {

src/IceCfg.h

@@ -111,6 +111,8 @@ public:
   void placePhiLoads();
   void placePhiStores();
   void deletePhis();
+  void advancedPhiLowering();
+  void reorderNodes();
   void doAddressOpt();
   void doArgLowering();
   void doNopInsertion();
@@ -120,6 +122,7 @@ public:
   void liveness(LivenessMode Mode);
   bool validateLiveness() const;
   void deleteRedundantAssignments();
+  void contractEmptyNodes();
   void doBranchOpt();
 
   // Manage the CurrentNode field, which is used for validating the

src/IceCfgNode.h

@@ -46,6 +46,9 @@ public:
   void setHasReturn() { HasReturn = true; }
   bool getHasReturn() const { return HasReturn; }
 
+  void setNeedsPlacement(bool Value) { NeedsPlacement = Value; }
+  bool needsPlacement() const { return NeedsPlacement; }
+
   // Access predecessor and successor edge lists.
   const NodeList &getInEdges() const { return InEdges; }
   const NodeList &getOutEdges() const { return OutEdges; }
@@ -64,16 +67,19 @@ public:
   // Add a predecessor edge to the InEdges list for each of this
   // node's successors.
   void computePredecessors();
+  CfgNode *splitIncomingEdge(CfgNode *Pred, SizeT InEdgeIndex);
 
   void placePhiLoads();
   void placePhiStores();
   void deletePhis();
+  void advancedPhiLowering();
   void doAddressOpt();
   void doNopInsertion();
   void genCode();
   void livenessLightweight();
   bool liveness(Liveness *Liveness);
   void livenessPostprocess(LivenessMode Mode, Liveness *Liveness);
+  void contractIfEmpty();
   void doBranchOpt(const CfgNode *NextNode);
   void emit(Cfg *Func) const;
   void dump(Cfg *Func) const;
@@ -84,6 +90,7 @@ private:
   const SizeT Number; // label index
   IceString Name;     // for dumping only
   bool HasReturn;     // does this block need an epilog?
+  bool NeedsPlacement;
   InstNumberT InstCountEstimate; // rough instruction count estimate
   NodeList InEdges;   // in no particular order
   NodeList OutEdges;  // in no particular order

src/IceDefs.h

@@ -42,6 +42,7 @@ class FunctionDeclaration;
 class GlobalContext;
 class GlobalDeclaration;
 class Inst;
+class InstAssign;
 class InstPhi;
 class InstTarget;
 class LiveRange;
@@ -56,6 +57,7 @@ class VariablesMetadata;
 // http://llvm.org/docs/ProgrammersManual.html#picking-the-right-data-structure-for-a-task
 typedef std::string IceString;
 typedef std::list<Inst *> InstList;
+typedef std::list<InstAssign *> AssignList;
 typedef std::list<InstPhi *> PhiList;
 typedef std::vector<Variable *> VarList;
 typedef std::vector<Operand *> OperandList;

src/IceInst.cpp

@@ -135,12 +135,12 @@ void Inst::livenessLightweight(Cfg *Func, LivenessBV &Live) {
   }
 }
 
-void Inst::liveness(InstNumberT InstNumber, LivenessBV &Live,
+bool Inst::liveness(InstNumberT InstNumber, LivenessBV &Live,
                     Liveness *Liveness, LiveBeginEndMap *LiveBegin,
                     LiveBeginEndMap *LiveEnd) {
   assert(!isDeleted());
   if (llvm::isa<InstFakeKill>(this))
-    return;
+    return true;
 
   Dead = false;
   if (Dest) {
@@ -158,7 +158,7 @@ void Inst::liveness(InstNumberT InstNumber, LivenessBV &Live,
     }
   }
   if (Dead)
-    return;
+    return false;
   // Phi arguments only get added to Live in the predecessor node, but
   // we still need to update LiveRangesEnded.
   bool IsPhi = llvm::isa<InstPhi>(this);
@@ -202,6 +202,7 @@ void Inst::liveness(InstNumberT InstNumber, LivenessBV &Live,
       }
     }
   }
+  return true;
 }
 
 InstAlloca::InstAlloca(Cfg *Func, Operand *ByteCount, uint32_t AlignInBytes,
@@ -261,6 +262,17 @@ NodeList InstBr::getTerminatorEdges() const {
   return OutEdges;
 }
 
+bool InstBr::repointEdge(CfgNode *OldNode, CfgNode *NewNode) {
+  if (TargetFalse == OldNode) {
+    TargetFalse = NewNode;
+    return true;
+  } else if (TargetTrue == OldNode) {
+    TargetTrue = NewNode;
+    return true;
+  }
+  return false;
+}
+
 InstCast::InstCast(Cfg *Func, OpKind CastKind, Variable *Dest, Operand *Source)
     : InstHighLevel(Func, Inst::Cast, 1, Dest), CastKind(CastKind) {
   addSource(Source);
@@ -410,6 +422,20 @@ NodeList InstSwitch::getTerminatorEdges() const {
   return OutEdges;
 }
 
+bool InstSwitch::repointEdge(CfgNode *OldNode, CfgNode *NewNode) {
+  if (LabelDefault == OldNode) {
+    LabelDefault = NewNode;
+    return true;
+  }
+  for (SizeT I = 0; I < NumCases; ++I) {
+    if (Labels[I] == OldNode) {
+      Labels[I] = NewNode;
+      return true;
+    }
+  }
+  return false;
+}
+
 InstUnreachable::InstUnreachable(Cfg *Func)
     : InstHighLevel(Func, Inst::Unreachable, 0, NULL) {}
 

src/IceInst.h

@@ -100,11 +100,27 @@ public:
         "getTerminatorEdges() called on a non-terminator instruction");
     return NodeList();
   }
+  virtual bool isUnconditionalBranch() const { return false; }
+  // If the instruction is a branch-type instruction with OldNode as a
+  // target, repoint it to NewNode and return true, otherwise return
+  // false.  Only repoint one instance, even if the instruction has
+  // multiple instances of OldNode as a target.
+  virtual bool repointEdge(CfgNode *OldNode, CfgNode *NewNode) {
+    (void)OldNode;
+    (void)NewNode;
+    return false;
+  }
 
   virtual bool isSimpleAssign() const { return false; }
 
   void livenessLightweight(Cfg *Func, LivenessBV &Live);
-  void liveness(InstNumberT InstNumber, LivenessBV &Live, Liveness *Liveness,
+  // Calculates liveness for this instruction.  Returns true if this
+  // instruction is (tentatively) still live and should be retained,
+  // and false if this instruction is (tentatively) dead and should be
+  // deleted.  The decision is tentative until the liveness dataflow
+  // algorithm has converged, and then a separate pass permanently
+  // deletes dead instructions.
+  bool liveness(InstNumberT InstNumber, LivenessBV &Live, Liveness *Liveness,
                 LiveBeginEndMap *LiveBegin, LiveBeginEndMap *LiveEnd);
 
   // Get the number of native instructions that this instruction
@@ -304,6 +320,8 @@ public:
     return getTargetFalse();
   }
   NodeList getTerminatorEdges() const override;
+  bool isUnconditionalBranch() const override { return isUnconditional(); }
+  bool repointEdge(CfgNode *OldNode, CfgNode *NewNode) override;
   void dump(const Cfg *Func) const override;
   static bool classof(const Inst *Inst) { return Inst->getKind() == Br; }
 
@@ -314,8 +332,8 @@ private:
   InstBr(Cfg *Func, CfgNode *Target);
   ~InstBr() override {}
 
-  CfgNode *const TargetFalse; // Doubles as unconditional branch target
-  CfgNode *const TargetTrue;  // NULL if unconditional branch
+  CfgNode *TargetFalse; // Doubles as unconditional branch target
+  CfgNode *TargetTrue;  // NULL if unconditional branch
 };
 
 // Call instruction.  The call target is captured as getSrc(0), and
@@ -671,6 +689,7 @@ public:
   }
   void addBranch(SizeT CaseIndex, uint64_t Value, CfgNode *Label);
   NodeList getTerminatorEdges() const override;
+  bool repointEdge(CfgNode *OldNode, CfgNode *NewNode) override;
   void dump(const Cfg *Func) const override;
   static bool classof(const Inst *Inst) { return Inst->getKind() == Switch; }
 

src/IceInstX8632.cpp

@@ -184,6 +184,17 @@ bool InstX8632Br::optimizeBranch(const CfgNode *NextNode) {
   return false;
 }
 
+bool InstX8632Br::repointEdge(CfgNode *OldNode, CfgNode *NewNode) {
+  if (TargetFalse == OldNode) {
+    TargetFalse = NewNode;
+    return true;
+  } else if (TargetTrue == OldNode) {
+    TargetTrue = NewNode;
+    return true;
+  }
+  return false;
+}
+
 InstX8632Call::InstX8632Call(Cfg *Func, Variable *Dest, Operand *CallTarget)
     : InstX8632(Func, InstX8632::Call, 1, Dest) {
   HasSideEffects = true;

src/IceInstX8632.h

@@ -387,6 +387,10 @@ public:
       ++Sum;
     return Sum;
   }
+  bool isUnconditionalBranch() const override {
+    return !Label && Condition == CondX86::Br_None;
+  }
+  bool repointEdge(CfgNode *OldNode, CfgNode *NewNode) override;
   void emit(const Cfg *Func) const override;
   void emitIAS(const Cfg *Func) const override;
   void dump(const Cfg *Func) const override;

src/IceLiveness.cpp

@@ -53,6 +53,7 @@ void Liveness::init() {
   for (SizeT i = 0; i < NumNodes; ++i) {
     Nodes[i].LiveToVarMap.assign(Nodes[i].NumLocals, NULL);
     Nodes[i].NumLocals = 0;
+    Nodes[i].NumNonDeadPhis = 0;
   }
   LiveToVarMap.assign(NumGlobals, NULL);
 

src/IceLiveness.h

@@ -32,9 +32,14 @@ class LivenessNode {
   // LivenessNode &operator=(const LivenessNode &) = delete;
 
 public:
-  LivenessNode() : NumLocals(0) {}
+  LivenessNode() : NumLocals(0), NumNonDeadPhis(0) {}
   // NumLocals is the number of Variables local to this block.
   SizeT NumLocals;
+  // NumNonDeadPhis tracks the number of Phi instructions that
+  // Inst::liveness() identified as tentatively live.  If
+  // NumNonDeadPhis changes from the last liveness pass, then liveness
+  // has not yet converged.
+  SizeT NumNonDeadPhis;
   // LiveToVarMap maps a liveness bitvector index to a Variable.  This
   // is generally just for printing/dumping.  The index should be less
   // than NumLocals + Liveness::NumGlobals.
@@ -66,20 +71,36 @@ public:
   SizeT getNumVarsInNode(const CfgNode *Node) const {
     return NumGlobals + Nodes[Node->getIndex()].NumLocals;
   }
+  SizeT &getNumNonDeadPhis(const CfgNode *Node) {
+    return Nodes[Node->getIndex()].NumNonDeadPhis;
+  }
   LivenessBV &getLiveIn(const CfgNode *Node) {
-    return Nodes[Node->getIndex()].LiveIn;
+    SizeT Index = Node->getIndex();
+    resize(Index);
+    return Nodes[Index].LiveIn;
   }
   LivenessBV &getLiveOut(const CfgNode *Node) {
-    return Nodes[Node->getIndex()].LiveOut;
+    SizeT Index = Node->getIndex();
+    resize(Index);
+    return Nodes[Index].LiveOut;
   }
   LiveBeginEndMap *getLiveBegin(const CfgNode *Node) {
-    return &Nodes[Node->getIndex()].LiveBegin;
+    SizeT Index = Node->getIndex();
+    resize(Index);
+    return &Nodes[Index].LiveBegin;
   }
   LiveBeginEndMap *getLiveEnd(const CfgNode *Node) {
-    return &Nodes[Node->getIndex()].LiveEnd;
+    SizeT Index = Node->getIndex();
+    resize(Index);
+    return &Nodes[Index].LiveEnd;
   }
 
 private:
+  // Resize Nodes so that Nodes[Index] is valid.
+  void resize(SizeT Index) {
+    if (Index >= Nodes.size())
+      Nodes.resize(Index + 1);
+  }
   Cfg *Func;
   LivenessMode Mode;
   SizeT NumGlobals;

src/IceOperand.cpp

@@ -320,38 +320,63 @@ void VariablesMetadata::init(MetadataKind TrackingKind) {
         .markUse(Kind, NoInst, EntryNode, IsFromDef, IsImplicit);
   }
 
-  for (CfgNode *Node : Func->getNodes()) {
-    for (Inst *I : Node->getInsts()) {
-      if (I->isDeleted())
-        continue;
-      if (InstFakeKill *Kill = llvm::dyn_cast<InstFakeKill>(I)) {
-        // A FakeKill instruction indicates certain Variables (usually
-        // physical scratch registers) are redefined, so we register
-        // them as defs.
-        for (SizeT SrcNum = 0; SrcNum < I->getSrcSize(); ++SrcNum) {
-          Variable *Var = llvm::cast<Variable>(I->getSrc(SrcNum));
-          SizeT VarNum = Var->getIndex();
-          assert(VarNum < Metadata.size());
-          Metadata[VarNum].markDef(Kind, Kill, Node);
-        }
-        continue; // no point in executing the rest
-      }
-      if (Variable *Dest = I->getDest()) {
-        SizeT DestNum = Dest->getIndex();
-        assert(DestNum < Metadata.size());
-        Metadata[DestNum].markDef(Kind, I, Node);
+  for (CfgNode *Node : Func->getNodes())
+    addNode(Node);
+}
+
+void VariablesMetadata::addNode(CfgNode *Node) {
+  if (Func->getNumVariables() >= Metadata.size())
+    Metadata.resize(Func->getNumVariables());
+
+  for (InstPhi *I : Node->getPhis()) {
+    if (I->isDeleted())
+      continue;
+    if (Variable *Dest = I->getDest()) {
+      SizeT DestNum = Dest->getIndex();
+      assert(DestNum < Metadata.size());
+      Metadata[DestNum].markDef(Kind, I, Node);
+    }
+    for (SizeT SrcNum = 0; SrcNum < I->getSrcSize(); ++SrcNum) {
+      if (const Variable *Var = llvm::dyn_cast<Variable>(I->getSrc(SrcNum))) {
+        SizeT VarNum = Var->getIndex();
+        assert(VarNum < Metadata.size());
+        const bool IsFromDef = false;
+        const bool IsImplicit = false;
+        Metadata[VarNum].markUse(Kind, I, Node, IsFromDef, IsImplicit);
       }
+    }
+  }
+
+  for (Inst *I : Node->getInsts()) {
+    if (I->isDeleted())
+      continue;
+    if (InstFakeKill *Kill = llvm::dyn_cast<InstFakeKill>(I)) {
+      // A FakeKill instruction indicates certain Variables (usually
+      // physical scratch registers) are redefined, so we register
+      // them as defs.
       for (SizeT SrcNum = 0; SrcNum < I->getSrcSize(); ++SrcNum) {
-        Operand *Src = I->getSrc(SrcNum);
-        SizeT NumVars = Src->getNumVars();
-        for (SizeT J = 0; J < NumVars; ++J) {
-          const Variable *Var = Src->getVar(J);
-          SizeT VarNum = Var->getIndex();
-          assert(VarNum < Metadata.size());
-          const bool IsFromDef = false;
-          const bool IsImplicit = false;
-          Metadata[VarNum].markUse(Kind, I, Node, IsFromDef, IsImplicit);
-        }
+        Variable *Var = llvm::cast<Variable>(I->getSrc(SrcNum));
+        SizeT VarNum = Var->getIndex();
+        assert(VarNum < Metadata.size());
+        Metadata[VarNum].markDef(Kind, Kill, Node);
+      }
+      continue; // no point in executing the rest
+    }
+    if (Variable *Dest = I->getDest()) {
+      SizeT DestNum = Dest->getIndex();
+      assert(DestNum < Metadata.size());
+      Metadata[DestNum].markDef(Kind, I, Node);
+    }
+    for (SizeT SrcNum = 0; SrcNum < I->getSrcSize(); ++SrcNum) {
+      Operand *Src = I->getSrc(SrcNum);
+      SizeT NumVars = Src->getNumVars();
+      for (SizeT J = 0; J < NumVars; ++J) {
+        const Variable *Var = Src->getVar(J);
+        SizeT VarNum = Var->getIndex();
+        assert(VarNum < Metadata.size());
+        const bool IsFromDef = false;
+        const bool IsImplicit = false;
+        Metadata[VarNum].markUse(Kind, I, Node, IsFromDef, IsImplicit);
       }
     }
   }

src/IceOperand.h

@@ -398,6 +398,8 @@ public:
   void setIgnoreLiveness() { IgnoreLiveness = true; }
   bool getIgnoreLiveness() const { return IgnoreLiveness; }
 
+  bool needsStackSlot() const { return NeedsStackSlot; }
+  void setNeedsStackSlot() { NeedsStackSlot = true; }
   int32_t getStackOffset() const { return StackOffset; }
   void setStackOffset(int32_t Offset) { StackOffset = Offset; }
 
@@ -474,9 +476,9 @@ public:
 protected:
   Variable(OperandKind K, Type Ty, SizeT Index, const IceString &Name)
       : Operand(K, Ty), Number(Index), Name(Name), IsArgument(false),
-        IsImplicitArgument(false), IgnoreLiveness(false), StackOffset(0),
-        RegNum(NoRegister), RegNumTmp(NoRegister), Weight(1), LoVar(NULL),
-        HiVar(NULL) {
+        IsImplicitArgument(false), IgnoreLiveness(false), NeedsStackSlot(false),
+        StackOffset(0), RegNum(NoRegister), RegNumTmp(NoRegister), Weight(1),
+        LoVar(NULL), HiVar(NULL) {
     Vars = VarsReal;
     Vars[0] = this;
     NumVars = 1;
@@ -492,6 +494,9 @@ protected:
   // constructing and validating live ranges.  This is usually
   // reserved for the stack pointer.
   bool IgnoreLiveness;
+  // NeedsStackSlot starts out false, and is set to true once we know
+  // for sure that the variable needs a stack slot.
+  bool NeedsStackSlot;
   // StackOffset is the canonical location on stack (only if
   // RegNum==NoRegister || IsArgument).
   int32_t StackOffset;
@@ -578,6 +583,9 @@ public:
   // Initialize the state by traversing all instructions/variables in
   // the CFG.
   void init(MetadataKind TrackingKind);
+  // Add a single node.  This is called by init(), and can be called
+  // incrementally from elsewhere, e.g. after edge-splitting.
+  void addNode(CfgNode *Node);
   // Returns whether the given Variable is tracked in this object.  It
   // should only return false if changes were made to the CFG after
   // running init(), in which case the state is stale and the results

src/IceRegAlloc.cpp

@@ -105,12 +105,15 @@ void LinearScan::scan(const llvm::SmallBitVector &RegMaskFull) {
     for (Variable *Var : Vars) {
       // Explicitly don't consider zero-weight variables, which are
       // meant to be spill slots.
-      if (Var->getWeight() == RegWeight::Zero)
+      if (Var->getWeight() == RegWeight::Zero) {
+        Var->setNeedsStackSlot();
         continue;
+      }
       // Don't bother if the variable has a null live range, which means
       // it was never referenced.
       if (Var->getLiveRange().isEmpty())
         continue;
+      Var->setNeedsStackSlot();
       Var->untrimLiveRange();
       Unhandled.push_back(Var);
       if (Var->hasReg()) {

src/IceTargetLowering.cpp

@@ -44,12 +44,16 @@ cl::opt<int> NopProbabilityAsPercentage(
 
 void LoweringContext::init(CfgNode *N) {
   Node = N;
+  End = getNode()->getInsts().end();
+  rewind();
+  advanceForward(Next);
+}
+
+void LoweringContext::rewind() {
   Begin = getNode()->getInsts().begin();
   Cur = Begin;
-  End = getNode()->getInsts().end();
   skipDeleted(Cur);
   Next = Cur;
-  advanceForward(Next);
 }
 
 void LoweringContext::insert(Inst *Inst) {

src/IceTargetLowering.h

@@ -64,6 +64,7 @@ public:
   Inst *getLastInserted() const;
   void advanceCur() { Cur = Next; }
   void advanceNext() { advanceForward(Next); }
+  void rewind();
   void setInsertPoint(const InstList::iterator &Position) { Next = Position; }
 
 private:
@@ -134,8 +135,18 @@ public:
   void doAddressOpt();
   // Randomly insert NOPs.
   void doNopInsertion();
-  // Lowers a single instruction.
+  // Lowers a single non-Phi instruction.
   void lower();
+  // Does preliminary lowering of the set of Phi instructions in the
+  // current node.  The main intention is to do what's needed to keep
+  // the unlowered Phi instructions consistent with the lowered
+  // non-Phi instructions, e.g. to lower 64-bit operands on a 32-bit
+  // target.
+  virtual void prelowerPhis() {}
+  // Lowers a list of "parallel" assignment instructions representing
+  // a topological sort of the Phi instructions.
+  virtual void lowerPhiAssignments(CfgNode *Node,
+                                   const AssignList &Assignments) = 0;
   // Tries to do branch optimization on a single instruction.  Returns
   // true if some optimization was done.
   virtual bool doBranchOpt(Inst * /*I*/, const CfgNode * /*NextNode*/) {

src/IceTargetLoweringX8632.h

@@ -105,6 +105,9 @@ protected:
   void lowerStore(const InstStore *Inst) override;
   void lowerSwitch(const InstSwitch *Inst) override;
   void lowerUnreachable(const InstUnreachable *Inst) override;
+  void prelowerPhis() override;
+  void lowerPhiAssignments(CfgNode *Node,
+                           const AssignList &Assignments) override;
   void doAddressOptLoad() override;
   void doAddressOptStore() override;
   void randomlyInsertNop(float Probability) override;
@@ -482,7 +485,7 @@ protected:
   llvm::SmallBitVector RegsUsed;
   SizeT NextLabelNumber;
   bool ComputedLiveRanges;
-  VarList PhysicalRegisters;
+  VarList PhysicalRegisters[IceType_NUM];
   static IceString RegNames[];
 
 private:

src/IceTimerTree.def

@@ -18,6 +18,7 @@
   /* enum value */         \
   X(O2)                    \
   X(Om1)                   \
+  X(advancedPhiLowering)   \
   X(convertToIce)          \
   X(deletePhis)            \
   X(doAddressOpt)          \

src/IceTypes.cpp

@@ -19,7 +19,7 @@ namespace Ice {
 namespace {
 
 const char *TargetArchName[] = {
-#define X(tag, str)  str ,
+#define X(tag, str) str,
   TARGETARCH_TABLE
 #undef X
 };

src/IceTypes.h

@@ -31,7 +31,7 @@ enum TargetArch {
 #define X(tag, str) tag,
   TARGETARCH_TABLE
 #undef X
-  TargetArch_NUM
+      TargetArch_NUM
 };
 
 const char *targetArchString(TargetArch Arch);

src/llvm2ice.cpp

@@ -169,10 +169,11 @@ static cl::opt<bool> AlwaysExitSuccess(
     "exit-success", cl::desc("Exit with success status, even if errors found"),
     cl::init(false));
 
-static cl::opt<bool> GenerateBuildAtts(
-    "build-atts", cl::desc("Generate list of build attributes associated with "
+static cl::opt<bool>
+GenerateBuildAtts("build-atts",
+                  cl::desc("Generate list of build attributes associated with "
                            "this executable."),
-    cl::init(false));
+                  cl::init(false));
 
 static int GetReturnValue(int Val) {
   if (AlwaysExitSuccess)
@@ -183,13 +184,12 @@ static int GetReturnValue(int Val) {
 static struct {
   const char *FlagName;
   int FlagValue;
-} ConditionalBuildAttributes[] = {
-  { "text_asm", ALLOW_TEXT_ASM },
-  { "dump", ALLOW_DUMP },
-  { "llvm_cl", ALLOW_LLVM_CL },
-  { "llvm_ir", ALLOW_LLVM_IR },
-  { "llvm_ir_as_input", ALLOW_LLVM_IR_AS_INPUT }
-};
+} ConditionalBuildAttributes[] = { { "text_asm", ALLOW_TEXT_ASM },
+                                   { "dump", ALLOW_DUMP },
+                                   { "llvm_cl", ALLOW_LLVM_CL },
+                                   { "llvm_ir", ALLOW_LLVM_IR },
+                                   { "llvm_ir_as_input",
+                                     ALLOW_LLVM_IR_AS_INPUT } };
 
 // Validates values of build attributes. Prints them to Stream if
 // Stream is non-null.

tests_lit/llvm2ice_tests/nacl-atomic-cmpxchg-optimization.ll

@@ -39,11 +39,7 @@ done:
 ; O2-LABEL: test_atomic_cmpxchg_loop
 ; O2: lock
 ; O2-NEXT: cmpxchg dword ptr [e{{[^a].}}], e{{[^a]}}
-; O2-NOT: cmp
-; Make sure the phi assignment for succeeded_first_try is still there.
-; O2: mov {{.*}}, 2
-; O2-NOT: cmp
-; O2: jne
+; O2-NEXT: j{{e|ne}}
 ; Make sure the call isn't accidentally deleted.
 ; O2: call
 ;
@@ -97,8 +93,7 @@ done:
 ; O2-LABEL: test_atomic_cmpxchg_loop_const
 ; O2: lock
 ; O2-NEXT: cmpxchg dword ptr [e{{[^a].}}], e{{[^a]}}
-; O2-NOT: cmp
-; O2: jne
+; O2-NEXT: j{{e|ne}}
 
 ; This is a case where the flags cannot be reused (compare is for some
 ; other condition).
@@ -120,7 +115,6 @@ done:
 ; O2-LABEL: test_atomic_cmpxchg_no_opt
 ; O2: lock
 ; O2-NEXT: cmpxchg dword ptr [e{{[^a].}}], e{{[^a]}}
-; O2: mov {{.*}}
 ; O2: cmp
 ; O2: jle
 

tests_lit/llvm2ice_tests/simple-loop.ll

@@ -35,14 +35,15 @@ for.end:
 ; CHECK-LABEL: simple_loop
 ; CHECK:      mov ecx, dword ptr [esp{{.*}}+{{.*}}{{[0-9]+}}]
 ; CHECK:      cmp ecx, 0
-; CHECK-NEXT: jle {{[0-9]}}
-
-; TODO: the mov from ebx to esi seems redundant here - so this may need to be
-; modified later
-
-; CHECK:      add [[IREG:[a-z]+]], 1
-; CHECK-NEXT: mov [[ICMPREG:[a-z]+]], [[IREG]]
-; CHECK:      cmp [[ICMPREG]], ecx
+; CHECK-NEXT: j{{le|g}} {{[0-9]}}
+
+; Check for the combination of address mode inference, register
+; allocation, and load/arithmetic fusing.
+; CHECK:      add e{{..}}, dword ptr [e{{..}} + 4*[[IREG:e..]]]
+; Check for incrementing of the register-allocated induction variable.
+; CHECK-NEXT: add [[IREG]], 1
+; Check for comparing the induction variable against the loop size.
+; CHECK-NEXT: cmp [[IREG]],
 ; CHECK-NEXT: jl -{{[0-9]}}
 ;
 ; There's nothing remarkable under Om1 to test for, since Om1 generates