Change the way to convert reverse operation When EVL lowering.

Author: Mel-Chen

llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h

@@ -245,7 +245,8 @@ struct Recipe_match {
     if constexpr (std::is_same<RecipeTy, VPScalarIVStepsRecipe>::value ||
                   std::is_same<RecipeTy, VPCanonicalIVPHIRecipe>::value ||
                   std::is_same<RecipeTy, VPDerivedIVRecipe>::value ||
-                  std::is_same<RecipeTy, VPWidenGEPRecipe>::value)
+                  std::is_same<RecipeTy, VPWidenGEPRecipe>::value ||
+                  std::is_same<RecipeTy, VPVectorEndPointerRecipe>::value)
       return DefR;
     else
       return DefR && DefR->getOpcode() == Opcode;
@@ -550,6 +551,16 @@ m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
   return m_Select(Op0, m_True(), Op1);
 }
 
+template <typename Op0_t, typename Op1_t>
+using VPVectorEndPointer_match =
+    Recipe_match<std::tuple<Op0_t, Op1_t>, 0, false, VPVectorEndPointerRecipe>;
+
+template <typename Op0_t, typename Op1_t>
+inline VPVectorEndPointer_match<Op0_t, Op1_t>
+m_VectorEndPointer(const Op0_t &Op0, const Op1_t &Op1) {
+  return VPVectorEndPointer_match<Op0_t, Op1_t>({Op0, Op1});
+}
+
 template <typename Op0_t, typename Op1_t, typename Op2_t>
 using VPScalarIVSteps_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0,
                                            false, VPScalarIVStepsRecipe>;

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -2151,6 +2151,22 @@ void VPlanTransforms::addActiveLaneMask(
   HeaderMask->eraseFromParent();
 }
 
+/// If \p R is a VPInstruction::Reverse, return a VPWidenIntrinsicRecipe
+/// for the vp.reverse intrinsic using \p EVL. Returns nullptr otherwise.
+static VPWidenIntrinsicRecipe *
+getEVLReverse(VPRecipeBase &R, VPTypeAnalysis &TypeInfo, VPValue &EVL) {
+  VPValue *ReversedVal;
+  if (!match(&R,
+             m_VPInstruction<VPInstruction::Reverse>(m_VPValue(ReversedVal))))
+    return nullptr;
+
+  auto *Reverse = cast<VPInstruction>(&R);
+  VPlan *Plan = Reverse->getParent()->getPlan();
+  return new VPWidenIntrinsicRecipe(
+      Intrinsic::experimental_vp_reverse, {ReversedVal, Plan->getTrue(), &EVL},
+      TypeInfo.inferScalarType(Reverse), Reverse->getDebugLoc());
+}
+
 /// Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
 /// EVL-based recipe without the header mask. Returns nullptr if no EVL-based
 /// recipe could be created.
@@ -2227,32 +2243,6 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
       .Default([&](VPRecipeBase *R) { return nullptr; });
 }
 
-static void convertToEVLReverse(VPlan &Plan, VPTypeAnalysis &TypeInfo,
-                                VPValue &AllOneMask, VPValue &EVL) {
-  SmallVector<VPRecipeBase *> ToRemove;
-
-  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
-           vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
-    for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
-      auto *VPI = dyn_cast<VPInstruction>(&R);
-      if (!VPI || VPI->getOpcode() != VPInstruction::Reverse)
-        continue;
-
-      SmallVector<VPValue *> Ops(VPI->operands());
-      Ops.append({&AllOneMask, &EVL});
-      auto *NewReverse = new VPWidenIntrinsicRecipe(
-          Intrinsic::experimental_vp_reverse, Ops,
-          TypeInfo.inferScalarType(VPI), VPI->getDebugLoc());
-      NewReverse->insertBefore(VPI);
-      VPI->replaceAllUsesWith(NewReverse);
-      ToRemove.push_back(VPI);
-    }
-  }
-
-  for (VPRecipeBase *R : ToRemove)
-    R->eraseFromParent();
-}
-
 /// Replace recipes with their EVL variants.
 static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
   VPTypeAnalysis TypeInfo(Plan);
@@ -2365,8 +2355,40 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
       CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue());
     }
     ToErase.push_back(CurRecipe);
+
+    // Convert general reverse operations on loaded values and stored values
+    // into vp.reverse, when the VPVectorEndPointerRecipe adjusting the access
+    // address uses EVL instead of VF.
+    // TODO: Extend conversion along the def-use/use-def chain, as reverse
+    // operations may be eliminated or moved in the future.
+    if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(EVLRecipe);
+        MemR && match(MemR->getAddr(),
+                      m_VectorEndPointer(m_VPValue(), m_Specific(&EVL)))) {
+      assert(MemR->isReverse() &&
+             "Only reverse access uses VPVectorEndPointerRecipe as address");
+      VPRecipeBase *Candidate = nullptr;
+      if (auto *LoadR = dyn_cast<VPWidenLoadEVLRecipe>(MemR)) {
+        assert(LoadR->getNumUsers() == 1 &&
+               "Unexpected user number of reverse load");
+        Candidate = cast<VPRecipeBase>(*LoadR->user_begin());
+      } else if (auto *StoreR = dyn_cast<VPWidenStoreEVLRecipe>(MemR)) {
+        VPValue *StoredVal = StoreR->getStoredValue();
+        // Skip if the stored value is not defined in the loop region.
+        if (StoredVal->isDefinedOutsideLoopRegions())
+          continue;
+        Candidate = StoredVal->getDefiningRecipe();
+      }
+      assert(Candidate && "Must have one reverse operation for reverse access");
+
+      VPWidenIntrinsicRecipe *NewReverse =
+          getEVLReverse(*Candidate, TypeInfo, EVL);
+      assert(NewReverse &&
+             "Unable to get an EVL reverse when tail folding by EVL");
+      NewReverse->insertBefore(Candidate);
+      cast<VPInstruction>(Candidate)->replaceAllUsesWith(NewReverse);
+      ToErase.push_back(Candidate);
+    }
   }
-  convertToEVLReverse(Plan, TypeInfo, *AllOneMask, EVL);
   // Remove dead EVL mask.
   if (EVLMask->getNumUsers() == 0)
     ToErase.push_back(EVLMask->getDefiningRecipe());