Create identity instruction when there are vector_sz-1 isomorphic instructions

Change-Id: Iba09dc8a4b7fe851b34c6405236ea72dd75a7953

Author: hassnaaHamdi

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

@@ -226,6 +226,8 @@ static const int MinScheduleRegionSize = 16;
 /// Maximum allowed number of operands in the PHI nodes.
 static const unsigned MaxPHINumOperands = 128;
 
+static SmallDenseMap<Value *, Value *> IdentityInstrsMp;
+
 /// Predicate for the element types that the SLP vectorizer supports.
 ///
 /// The most important thing to filter here are types which are invalid in LLVM
@@ -2075,6 +2077,55 @@ class BoUpSLP {
 
   OptimizationRemarkEmitter *getORE() { return ORE; }
 
+  static SmallVector<Value*, 8> setIdentityInstr(ArrayRef<Value *> VL) {
+    SmallVector<Value *, 8> New_VL(VL.begin(), VL.end());
+   if (VL.size() <= 2)
+     return New_VL;
+    auto It = find_if(VL, IsaPred<Instruction>);
+    if (It == VL.end())
+      return New_VL;
+    // work on unique list of instructions only:
+    SmallDenseMap<llvm::StringRef, bool> SeenInstrs;
+    for (auto *V : VL)
+      if (auto *I = dyn_cast<Instruction>(V)) {
+        if (!SeenInstrs[I->getName()])
+          SeenInstrs[I->getName()] = true;
+        else {
+          return New_VL;
+        }
+      }
+    Instruction *MainOp = cast<Instruction>(*It);
+    auto ValidOperands = count_if(VL, IsaPred<Instruction, PoisonValue>);
+    if (ValidOperands != (int) VL.size()-1)
+      return New_VL;
+    auto DifferentOperand = find_if_not(VL, IsaPred<Instruction, PoisonValue>);
+    if (DifferentOperand == VL.end())
+      return New_VL;
+    assert(!isa<Instruction>(*DifferentOperand) && !isa<PoisonValue>(*DifferentOperand) &&
+    "Expected different operand to be not an instruction");
+    auto FoundIdentityInstrIt = IdentityInstrsMp.find(*DifferentOperand);
+    if (FoundIdentityInstrIt != IdentityInstrsMp.end()) {
+      auto OperandIndex = std::distance(VL.begin(), DifferentOperand);
+      New_VL[OperandIndex] = FoundIdentityInstrIt->second;
+      return New_VL;
+    }
+    auto *Identity = ConstantExpr::getIdentity(MainOp, MainOp->getType(), true /*AllowRHSConstant*/);
+    if (!Identity)
+      return New_VL;
+    auto *NewInstr = MainOp->clone();
+    NewInstr->setOperand(0, *DifferentOperand);
+    NewInstr->setOperand(1, Identity);
+    NewInstr->insertAfter(cast<Instruction>(MainOp));
+    NewInstr->setName((*DifferentOperand)->getName() + ".identity");
+    auto OperandIndex = std::distance(VL.begin(), DifferentOperand);
+    New_VL[OperandIndex] = NewInstr;
+    assert(find_if_not(New_VL, IsaPred<Instruction, PoisonValue>) ==
+      New_VL.end() &&
+          "Expected all operands to be instructions");
+    IdentityInstrsMp.try_emplace(*DifferentOperand, NewInstr);
+    return New_VL;
+  }
+
   /// This structure holds any data we need about the edges being traversed
   /// during buildTreeRec(). We keep track of:
   /// (i) the user TreeEntry index, and
@@ -3786,7 +3837,8 @@ class BoUpSLP {
       assert(OpVL.size() <= Scalars.size() &&
              "Number of operands is greater than the number of scalars.");
       Operands[OpIdx].resize(OpVL.size());
-      copy(OpVL, Operands[OpIdx].begin());
+      auto NewVL = BoUpSLP::setIdentityInstr(OpVL);
+      copy(NewVL, Operands[OpIdx].begin());
     }
 
   public:
@@ -4084,18 +4136,19 @@ class BoUpSLP {
         "Reshuffling scalars not yet supported for nodes with padding");
     Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
                                      ReuseShuffleIndices.end());
+    SmallVector<Value*, 8> NewVL =BoUpSLP::setIdentityInstr(VL);
     if (ReorderIndices.empty()) {
-      Last->Scalars.assign(VL.begin(), VL.end());
+      Last->Scalars.assign(NewVL.begin(), NewVL.end());
       if (S)
         Last->setOperations(S);
     } else {
       // Reorder scalars and build final mask.
-      Last->Scalars.assign(VL.size(), nullptr);
+      Last->Scalars.assign(NewVL.size(), nullptr);
       transform(ReorderIndices, Last->Scalars.begin(),
-                [VL](unsigned Idx) -> Value * {
-                  if (Idx >= VL.size())
-                    return UndefValue::get(VL.front()->getType());
-                  return VL[Idx];
+                [NewVL](unsigned Idx) -> Value * {
+                  if (Idx >= NewVL.size())
+                    return UndefValue::get(NewVL.front()->getType());
+                  return NewVL[Idx];
                 });
       InstructionsState S = getSameOpcode(Last->Scalars, *TLI);
       if (S)
@@ -4106,7 +4159,7 @@ class BoUpSLP {
       assert(S && "Split nodes must have operations.");
       Last->setOperations(S);
       SmallPtrSet<Value *, 4> Processed;
-      for (Value *V : VL) {
+      for (Value *V : NewVL) {
         auto *I = dyn_cast<Instruction>(V);
         if (!I)
           continue;
@@ -4121,10 +4174,10 @@ class BoUpSLP {
         }
       }
     } else if (!Last->isGather()) {
-      if (doesNotNeedToSchedule(VL))
+      if (doesNotNeedToSchedule(NewVL))
         Last->setDoesNotNeedToSchedule();
       SmallPtrSet<Value *, 4> Processed;
-      for (Value *V : VL) {
+      for (Value *V : NewVL) {
         if (isa<PoisonValue>(V))
           continue;
         auto It = ScalarToTreeEntries.find(V);
@@ -4146,7 +4199,7 @@ class BoUpSLP {
 #if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
         auto *BundleMember = Bundle.getBundle().begin();
         SmallPtrSet<Value *, 4> Processed;
-        for (Value *V : VL) {
+        for (Value *V : NewVL) {
           if (doesNotNeedToBeScheduled(V) || !Processed.insert(V).second)
             continue;
           ++BundleMember;
@@ -4159,7 +4212,7 @@ class BoUpSLP {
     } else {
       // Build a map for gathered scalars to the nodes where they are used.
       bool AllConstsOrCasts = true;
-      for (Value *V : VL)
+      for (Value *V : NewVL)
         if (!isConstant(V)) {
           auto *I = dyn_cast<CastInst>(V);
           AllConstsOrCasts &= I && I->getType()->isIntegerTy();
@@ -4170,7 +4223,7 @@ class BoUpSLP {
       if (AllConstsOrCasts)
         CastMaxMinBWSizes =
             std::make_pair(std::numeric_limits<unsigned>::max(), 1);
-      MustGather.insert_range(VL);
+      MustGather.insert_range(NewVL);
     }
 
     if (UserTreeIdx.UserTE)
@@ -20844,6 +20897,11 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
     }
   }
 
+  for (auto &I : IdentityInstrsMp) {
+    if (I.second && cast<Instruction>(I.second)->getParent())
+      cast<Instruction>(I.second)->eraseFromParent();
+  }
+  IdentityInstrsMp.clear();
   if (Changed) {
     R.optimizeGatherSequence();
     LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");

llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll

@@ -7,13 +7,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define <4 x i32> @foo(<4 x i32> %x, i32 %f) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <4 x i32> poison, i32 [[F:%.*]], i64 0
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[F]], 1
-; CHECK-NEXT:    [[VECINIT1:%.*]] = insertelement <4 x i32> [[VECINIT]], i32 [[ADD]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 2, i32 3>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[VECINIT51:%.*]] = shufflevector <4 x i32> [[VECINIT1]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[VECINIT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[VECINIT51:%.*]] = add nsw <4 x i32> [[TMP2]], <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x i32> [[VECINIT51]]
 ;
   %vecinit = insertelement <4 x i32> undef, i32 %f, i32 0

llvm/test/Transforms/SLPVectorizer/infer-missing-instruction.ll

@@ -5,24 +5,9 @@ define dso_local noundef i32 @_Z4testiPs(i32 noundef %a, ptr noundef readonly ca
 ; CHECK-LABEL: define dso_local noundef i32 @_Z4testiPs(
 ; CHECK-SAME: i32 noundef [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]]) local_unnamed_addr {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SHR_1:%.*]] = lshr i32 [[A]], 1
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[A]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i32> [[TMP1]], <i32 2, i32 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr <4 x i32> [[TMP4]], <i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> poison, i32 [[A]], i64 0
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = lshr <8 x i32> [[TMP7]], <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> poison, i32 [[A]], i64 0
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_1]], i64 1
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <16 x i32> [[TMP12]], <16 x i32> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i32> [[TMP14]], <16 x i32> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP9]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = lshr <16 x i32> [[TMP1]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP17:%.*]] = and <16 x i32> [[TMP16]], splat (i32 16)
 ; CHECK-NEXT:    [[TMP18:%.*]] = load <16 x i16>, ptr [[B]], align 2
 ; CHECK-NEXT:    [[TMP19:%.*]] = sext <16 x i16> [[TMP18]] to <16 x i32>