Subzero. Rematerializes shufflevector instructions.

src/IceCfg.h

@@ -206,6 +206,9 @@ public:
   /// entry block and emit stack or frame pointer-relative addressing.
   void processAllocas(bool SortAndCombine);
   void doAddressOpt();
+  /// Find clusters of insertelement/extractelement instructions that can be
+  /// replaced by a shufflevector instruction.
+  void materializeVectorShuffles();
   void doArgLowering();
   void doNopInsertion();
   void genCode();

src/IceClFlags.def

@@ -327,6 +327,8 @@ struct dev_list_flag {};
         clEnumValN(Ice::IceV_RMW, "rmw", "ReadModifyWrite optimization"),      \
         clEnumValN(Ice::IceV_Loop, "loop", "Loop nest depth analysis"),        \
         clEnumValN(Ice::IceV_Mem, "mem", "Memory usage details"),              \
+        clEnumValN(Ice::IceV_ShufMat, "shufvec",                               \
+	           "Shufflevector rematerialization"),                         \
         clEnumValN(Ice::IceV_Status, "status",                                 \
                    "Print the name of the function being translated"),         \
         clEnumValN(Ice::IceV_AvailableRegs, "registers",                       \

src/IceDefs.h

@@ -342,6 +342,7 @@ enum VerboseItem {
   IceV_GlobalInit = 1 << 22,
   IceV_ConstPoolStats = 1 << 23,
   IceV_Wasm = 1 << 24,
+  IceV_ShufMat = 1 << 25,
   IceV_All = ~IceV_None,
   IceV_Most =
       IceV_All & ~IceV_LinearScan & ~IceV_GlobalInit & ~IceV_ConstPoolStats

src/IceInst.cpp

@@ -112,6 +112,7 @@ const char *Inst::getInstName() const {
     X(FakeUse, "fakeuse");
     X(FakeKill, "fakekill");
     X(JumpTable, "jumptable");
+    X(ShuffleVector, "shufflevector");
 #undef X
   default:
     assert(Kind >= Target);
@@ -574,6 +575,15 @@ InstFakeUse::InstFakeUse(Cfg *Func, Variable *Src, uint32_t Weight)
 InstFakeKill::InstFakeKill(Cfg *Func, const Inst *Linked)
     : InstHighLevel(Func, Inst::FakeKill, 0, nullptr), Linked(Linked) {}
 
+InstShuffleVector::InstShuffleVector(Cfg *Func, Variable *Dest, Variable *Src0,
+                                     Variable *Src1)
+    : InstHighLevel(Func, Inst::ShuffleVector, 2, Dest),
+      NumIndexes(typeNumElements(Dest->getType())) {
+  addSource(Src0);
+  addSource(Src1);
+  Indexes = Func->allocateArrayOf<ConstantInteger32 *>(NumIndexes);
+}
+
 namespace {
 GlobalString makeName(Cfg *Func, const SizeT Id) {
   const auto FuncName = Func->getFunctionName();
@@ -1032,6 +1042,21 @@ void InstFakeKill::dump(const Cfg *Func) const {
   Str << "kill.pseudo scratch_regs";
 }
 
+void InstShuffleVector::dump(const Cfg *Func) const {
+  if (!BuildDefs::dump())
+    return;
+  Ostream &Str = Func->getContext()->getStrDump();
+  Str << "shufflevector ";
+  dumpDest(Func);
+  Str << " = ";
+  dumpSources(Func);
+  for (SizeT I = 0; I < NumIndexes; ++I) {
+    Str << ", ";
+    Indexes[I]->dump(Func);
+  }
+  Str << "\n";
+}
+
 void InstJumpTable::dump(const Cfg *Func) const {
   if (!BuildDefs::dump())
     return;

src/IceInst.h

@@ -22,6 +22,7 @@
 #include "IceDefs.h"
 #include "IceInst.def"
 #include "IceIntrinsics.h"
+#include "IceOperand.h"
 #include "IceSwitchLowering.h"
 #include "IceTypes.h"
 
@@ -61,14 +62,15 @@ public:
     Select,
     Store,
     Switch,
-    Assign,       // not part of LLVM/PNaCl bitcode
-    Breakpoint,   // not part of LLVM/PNaCl bitcode
-    BundleLock,   // not part of LLVM/PNaCl bitcode
-    BundleUnlock, // not part of LLVM/PNaCl bitcode
-    FakeDef,      // not part of LLVM/PNaCl bitcode
-    FakeUse,      // not part of LLVM/PNaCl bitcode
-    FakeKill,     // not part of LLVM/PNaCl bitcode
-    JumpTable,    // not part of LLVM/PNaCl bitcode
+    Assign,        // not part of LLVM/PNaCl bitcode
+    Breakpoint,    // not part of LLVM/PNaCl bitcode
+    BundleLock,    // not part of LLVM/PNaCl bitcode
+    BundleUnlock,  // not part of LLVM/PNaCl bitcode
+    FakeDef,       // not part of LLVM/PNaCl bitcode
+    FakeUse,       // not part of LLVM/PNaCl bitcode
+    FakeKill,      // not part of LLVM/PNaCl bitcode
+    JumpTable,     // not part of LLVM/PNaCl bitcode
+    ShuffleVector, // not part of LLVM/PNaCl bitcode
     // Anything >= Target is an InstTarget subclass. Note that the value-spaces
     // are shared across targets. To avoid confusion over the definition of
     // shared values, an object specific to one target should never be passed
@@ -917,6 +919,52 @@ private:
   const Inst *Linked;
 };
 
+/// ShuffleVector instruction. This represents a shuffle operation on vector
+/// types. This instruction is not part of the PNaCl bitcode: it is generated
+/// by Subzero when it matches the pattern used by pnacl-clang when compiling
+/// to bitcode.
+class InstShuffleVector : public InstHighLevel {
+  InstShuffleVector() = delete;
+  InstShuffleVector(const InstShuffleVector &) = delete;
+  InstShuffleVector &operator=(const InstShuffleVector &) = delete;
+
+public:
+  static InstShuffleVector *create(Cfg *Func, Variable *Dest, Variable *Src0,
+                                   Variable *Src1) {
+    return new (Func->allocate<InstShuffleVector>())
+        InstShuffleVector(Func, Dest, Src0, Src1);
+  }
+
+  SizeT getNumIndexes() const { return NumIndexes; }
+
+  void addIndex(ConstantInteger32 *Index) {
+    assert(CurrentIndex < NumIndexes);
+    Indexes[CurrentIndex++] = Index;
+  }
+
+  ConstantInteger32 *getIndex(SizeT Pos) const {
+    assert(Pos < NumIndexes);
+    return Indexes[Pos];
+  }
+
+  void dump(const Cfg *Func) const override;
+  static bool classof(const Inst *Instr) {
+    return Instr->getKind() == ShuffleVector;
+  }
+
+private:
+  InstShuffleVector(Cfg *Func, Variable *Dest, Variable *Src0, Variable *Src1);
+
+  void destroy(Cfg *Func) override {
+    Func->deallocateArrayOf<ConstantInteger32 *>(Indexes);
+    Inst::destroy(Func);
+  }
+
+  ConstantInteger32 **Indexes;
+  SizeT CurrentIndex = 0;
+  const SizeT NumIndexes;
+};
+
 /// JumpTable instruction. This represents a jump table that will be stored in
 /// the .rodata section. This is used to track and repoint the target CfgNodes
 /// which may change, for example due to splitting for phi lowering.

src/IceTargetLowering.cpp

@@ -440,6 +440,9 @@ void TargetLowering::lower() {
     case Inst::Select:
       lowerSelect(llvm::cast<InstSelect>(Instr));
       break;
+    case Inst::ShuffleVector:
+      lowerShuffleVector(llvm::cast<InstShuffleVector>(Instr));
+      break;
     case Inst::Store:
       lowerStore(llvm::cast<InstStore>(Instr));
       break;

src/IceTargetLowering.h

@@ -387,6 +387,7 @@ protected:
   virtual void lowerPhi(const InstPhi *Instr) = 0;
   virtual void lowerRet(const InstRet *Instr) = 0;
   virtual void lowerSelect(const InstSelect *Instr) = 0;
+  virtual void lowerShuffleVector(const InstShuffleVector *Instr) = 0;
   virtual void lowerStore(const InstStore *Instr) = 0;
   virtual void lowerSwitch(const InstSwitch *Instr) = 0;
   virtual void lowerUnreachable(const InstUnreachable *Instr) = 0;

src/IceTargetLoweringARM32.cpp

@@ -1020,6 +1020,7 @@ void TargetARM32::translateO2() {
   // Address mode optimization.
   Func->getVMetadata()->init(VMK_SingleDefs);
   Func->doAddressOpt();
+  Func->materializeVectorShuffles();
 
   // Argument lowering
   Func->doArgLowering();
@@ -5812,6 +5813,44 @@ void TargetARM32::lowerRet(const InstRet *Instr) {
   Context.insert<InstFakeUse>(SP);
 }
 
+void TargetARM32::lowerShuffleVector(const InstShuffleVector *Instr) {
+  auto *Dest = Instr->getDest();
+  const Type DestTy = Dest->getType();
+
+  auto *T = makeReg(DestTy);
+
+  switch (DestTy) {
+  default:
+    break;
+    // TODO(jpp): figure out how to properly lower this without scalarization.
+  }
+
+  // Unoptimized shuffle. Perform a series of inserts and extracts.
+  Context.insert<InstFakeDef>(T);
+  auto *Src0 = llvm::cast<Variable>(Instr->getSrc(0));
+  auto *Src1 = llvm::cast<Variable>(Instr->getSrc(1));
+  const SizeT NumElements = typeNumElements(DestTy);
+  const Type ElementType = typeElementType(DestTy);
+  for (SizeT I = 0; I < Instr->getNumIndexes(); ++I) {
+    auto *Index = Instr->getIndex(I);
+    const SizeT Elem = Index->getValue();
+    auto *ExtElmt = makeReg(ElementType);
+    if (Elem < NumElements) {
+      lowerExtractElement(
+          InstExtractElement::create(Func, ExtElmt, Src0, Index));
+    } else {
+      lowerExtractElement(InstExtractElement::create(
+          Func, ExtElmt, Src1,
+          Ctx->getConstantInt32(Index->getValue() - NumElements)));
+    }
+    auto *NewT = makeReg(DestTy);
+    lowerInsertElement(InstInsertElement::create(Func, NewT, T, ExtElmt,
+                                                 Ctx->getConstantInt32(I)));
+    T = NewT;
+  }
+  _mov(Dest, T);
+}
+
 void TargetARM32::lowerSelect(const InstSelect *Instr) {
   Variable *Dest = Instr->getDest();
   Type DestTy = Dest->getType();

src/IceTargetLoweringARM32.h

@@ -285,6 +285,7 @@ protected:
   void lowerPhi(const InstPhi *Instr) override;
   void lowerRet(const InstRet *Instr) override;
   void lowerSelect(const InstSelect *Instr) override;
+  void lowerShuffleVector(const InstShuffleVector *Instr) override;
   void lowerStore(const InstStore *Instr) override;
   void lowerSwitch(const InstSwitch *Instr) override;
   void lowerUnreachable(const InstUnreachable *Instr) override;

src/IceTargetLoweringMIPS32.cpp

@@ -1145,6 +1145,10 @@ void TargetMIPS32::lowerSelect(const InstSelect *Instr) {
   UnimplementedLoweringError(this, Instr);
 }
 
+void TargetMIPS32::lowerShuffleVector(const InstShuffleVector *Instr) {
+  UnimplementedLoweringError(this, Instr);
+}
+
 void TargetMIPS32::lowerStore(const InstStore *Instr) {
   UnimplementedLoweringError(this, Instr);
 }

src/IceTargetLoweringMIPS32.h

@@ -299,6 +299,7 @@ protected:
   void lowerPhi(const InstPhi *Instr) override;
   void lowerRet(const InstRet *Instr) override;
   void lowerSelect(const InstSelect *Instr) override;
+  void lowerShuffleVector(const InstShuffleVector *Instr) override;
   void lowerStore(const InstStore *Instr) override;
   void lowerSwitch(const InstSwitch *Instr) override;
   void lowerUnreachable(const InstUnreachable *Instr) override;

src/IceTargetLoweringX86Base.h

@@ -260,6 +260,7 @@ protected:
   void lowerPhi(const InstPhi *Instr) override;
   void lowerRet(const InstRet *Instr) override;
   void lowerSelect(const InstSelect *Instr) override;
+  void lowerShuffleVector(const InstShuffleVector *Instr) override;
   void lowerStore(const InstStore *Instr) override;
   void lowerSwitch(const InstSwitch *Instr) override;
   void lowerUnreachable(const InstUnreachable *Instr) override;

src/IceTargetLoweringX86BaseImpl.h

@@ -428,6 +428,7 @@ template <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() {
   // Address mode optimization.
   Func->getVMetadata()->init(VMK_SingleDefs);
   Func->doAddressOpt();
+  Func->materializeVectorShuffles();
 
   // Find read-modify-write opportunities. Do this after address mode
   // optimization so that doAddressOpt() doesn't need to be applied to RMW
@@ -5570,6 +5571,46 @@ void TargetX86Base<TraitsType>::lowerRet(const InstRet *Instr) {
 }
 
 template <typename TraitsType>
+void TargetX86Base<TraitsType>::lowerShuffleVector(
+    const InstShuffleVector *Instr) {
+  auto *Dest = Instr->getDest();
+  const Type DestTy = Dest->getType();
+
+  auto *T = makeReg(DestTy);
+
+  switch (DestTy) {
+  default:
+    break;
+    // TODO(jpp): figure out how to properly lower this without scalarization.
+  }
+
+  // Unoptimized shuffle. Perform a series of inserts and extracts.
+  Context.insert<InstFakeDef>(T);
+  auto *Src0 = llvm::cast<Variable>(Instr->getSrc(0));
+  auto *Src1 = llvm::cast<Variable>(Instr->getSrc(1));
+  const SizeT NumElements = typeNumElements(DestTy);
+  const Type ElementType = typeElementType(DestTy);
+  for (SizeT I = 0; I < Instr->getNumIndexes(); ++I) {
+    auto *Index = Instr->getIndex(I);
+    const SizeT Elem = Index->getValue();
+    auto *ExtElmt = makeReg(ElementType);
+    if (Elem < NumElements) {
+      lowerExtractElement(
+          InstExtractElement::create(Func, ExtElmt, Src0, Index));
+    } else {
+      lowerExtractElement(InstExtractElement::create(
+          Func, ExtElmt, Src1,
+          Ctx->getConstantInt32(Index->getValue() - NumElements)));
+    }
+    auto *NewT = makeReg(DestTy);
+    lowerInsertElement(InstInsertElement::create(Func, NewT, T, ExtElmt,
+                                                 Ctx->getConstantInt32(I)));
+    T = NewT;
+  }
+  _movp(Dest, T);
+}
+
+template <typename TraitsType>
 void TargetX86Base<TraitsType>::lowerSelect(const InstSelect *Select) {
   Variable *Dest = Select->getDest();
 

src/IceTimerTree.def

@@ -15,55 +15,56 @@
 #ifndef SUBZERO_SRC_ICETIMERTREE_DEF
 #define SUBZERO_SRC_ICETIMERTREE_DEF
 
-#define TIMERTREE_TABLE        \
-  /* enum value */             \
-  X(O2)                        \
-  X(Om1)                       \
-  X(advancedPhiLowering)       \
-  X(alloca)                    \
-  X(computeLoopNestDepth)      \
-  X(convertToIce)              \
-  X(deletePhis)                \
-  X(doAddressOpt)              \
-  X(doArgLowering)             \
-  X(doBranchOpt)               \
-  X(doNopInsertion)            \
-  X(emitAsm)                   \
-  X(emitGlobalInitializers)    \
-  X(findRMW)                   \
-  X(genCode)                   \
-  X(genFrame)                  \
-  X(genHelpers)                \
-  X(initUnhandled)             \
-  X(linearScan)                \
-  X(liveRange)                 \
-  X(liveness)                  \
-  X(livenessLightweight)       \
-  X(llvmConvert)               \
-  X(loadOpt)                   \
-  X(lowerPhiAssignments)       \
-  X(parse)                     \
-  X(parseConstants)            \
-  X(parseFunctions)            \
-  X(parseFunctionValuesymtabs) \
-  X(parseGlobals)              \
-  X(parseModule)               \
-  X(parseModuleValuesymtabs)   \
-  X(parseTypes)                \
-  X(phiValidation)             \
-  X(placePhiLoads)             \
-  X(placePhiStores)            \
-  X(qEmitPop)                  \
-  X(qEmitPush)                 \
-  X(qTransPop)                 \
-  X(qTransPush)                \
-  X(regAlloc)                  \
-  X(renumberInstructions)      \
-  X(szmain)                    \
-  X(translate)                 \
-  X(translateFunctions)        \
-  X(validateLiveness)          \
-  X(vmetadata)                 \
+#define TIMERTREE_TABLE                                                        \
+  /* enum value */                                                             \
+  X(O2)                                                                        \
+  X(Om1)                                                                       \
+  X(advancedPhiLowering)                                                       \
+  X(alloca)                                                                    \
+  X(computeLoopNestDepth)                                                      \
+  X(convertToIce)                                                              \
+  X(deletePhis)                                                                \
+  X(doAddressOpt)                                                              \
+  X(doArgLowering)                                                             \
+  X(doBranchOpt)                                                               \
+  X(doNopInsertion)                                                            \
+  X(emitAsm)                                                                   \
+  X(emitGlobalInitializers)                                                    \
+  X(findRMW)                                                                   \
+  X(genCode)                                                                   \
+  X(genFrame)                                                                  \
+  X(genHelpers)                                                                \
+  X(initUnhandled)                                                             \
+  X(linearScan)                                                                \
+  X(liveRange)                                                                 \
+  X(liveness)                                                                  \
+  X(livenessLightweight)                                                       \
+  X(llvmConvert)                                                               \
+  X(loadOpt)                                                                   \
+  X(lowerPhiAssignments)                                                       \
+  X(materializeVectorShuffles)                                                 \
+  X(parse)                                                                     \
+  X(parseConstants)                                                            \
+  X(parseFunctions)                                                            \
+  X(parseFunctionValuesymtabs)                                                 \
+  X(parseGlobals)                                                              \
+  X(parseModule)                                                               \
+  X(parseModuleValuesymtabs)                                                   \
+  X(parseTypes)                                                                \
+  X(phiValidation)                                                             \
+  X(placePhiLoads)                                                             \
+  X(placePhiStores)                                                            \
+  X(qEmitPop)                                                                  \
+  X(qEmitPush)                                                                 \
+  X(qTransPop)                                                                 \
+  X(qTransPush)                                                                \
+  X(regAlloc)                                                                  \
+  X(renumberInstructions)                                                      \
+  X(szmain)                                                                    \
+  X(translate)                                                                 \
+  X(translateFunctions)                                                        \
+  X(validateLiveness)                                                          \
+  X(vmetadata)                                                                 \
   X(writeELF)
 //#define X(tag)