[VPlan] Thread DL, use TargetFolder

Author: artagnon

llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

@@ -76,10 +76,16 @@ class VPBuilder {
   }
 
 public:
-  VPBuilder() = default;
-  VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); }
-  VPBuilder(VPRecipeBase *InsertPt) { setInsertPoint(InsertPt); }
-  VPBuilder(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) {
+  VPBuilder(const DataLayout &DL) : Folder(DL) {}
+  VPBuilder(const DataLayout &DL, VPBasicBlock *InsertBB) : Folder(DL) {
+    setInsertPoint(InsertBB);
+  }
+  VPBuilder(const DataLayout &DL, VPRecipeBase *InsertPt) : Folder(DL) {
+    setInsertPoint(InsertPt);
+  }
+  VPBuilder(const DataLayout &DL, VPBasicBlock *TheBB,
+            VPBasicBlock::iterator IP)
+      : Folder(DL) {
     setInsertPoint(TheBB, IP);
   }
 
@@ -94,8 +100,8 @@ class VPBuilder {
   VPBasicBlock::iterator getInsertPoint() const { return InsertPt; }
 
   /// Create a VPBuilder to insert after \p R.
-  static VPBuilder getToInsertAfter(VPRecipeBase *R) {
-    VPBuilder B;
+  static VPBuilder getToInsertAfter(const DataLayout &DL, VPRecipeBase *R) {
+    VPBuilder B(DL);
     B.setInsertPoint(R->getParent(), std::next(R->getIterator()));
     return B;
   }
@@ -458,9 +464,9 @@ class LoopVectorizationPlanner {
       const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
       LoopVectorizationCostModel &CM, InterleavedAccessInfo &IAI,
       PredicatedScalarEvolution &PSE, const LoopVectorizeHints &Hints,
-      OptimizationRemarkEmitter *ORE)
+      OptimizationRemarkEmitter *ORE, const DataLayout &DL)
       : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM),
-        IAI(IAI), PSE(PSE), Hints(Hints), ORE(ORE) {}
+        IAI(IAI), PSE(PSE), Hints(Hints), ORE(ORE), Builder(DL) {}
 
   /// Build VPlans for the specified \p UserVF and \p UserIC if they are
   /// non-zero or all applicable candidate VFs otherwise. If vectorization and

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -7698,7 +7698,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   // cost model is complete for better cost estimates.
   VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF,
                            OrigLoop->getHeader()->getContext());
-  VPlanTransforms::materializeBroadcasts(BestVPlan);
+  VPlanTransforms::materializeBroadcasts(
+      BestVPlan, OrigLoop->getHeader()->getDataLayout());
   VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
   VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
   VPlanTransforms::narrowInterleaveGroups(
@@ -8991,7 +8992,8 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
       // Discard the plan if it is not EVL-compatible
       if (CM.foldTailWithEVL() && !HasScalarVF &&
           !VPlanTransforms::runPass(VPlanTransforms::tryAddExplicitVectorLength,
-                                    *Plan, CM.getMaxSafeElements()))
+                                    *Plan, CM.getMaxSafeElements(),
+                                    OrigLoop->getHeader()->getDataLayout()))
         break;
       assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
       VPlans.push_back(std::move(Plan));
@@ -9003,7 +9005,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
 // Add the necessary canonical IV and branch recipes required to control the
 // loop.
 static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
-                                  DebugLoc DL) {
+                                  DebugLoc DL, const DataLayout &Layout) {
   Value *StartIdx = ConstantInt::get(IdxTy, 0);
   auto *StartV = Plan.getOrAddLiveIn(StartIdx);
 
@@ -9013,7 +9015,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
   VPBasicBlock *Header = TopRegion->getEntryBasicBlock();
   Header->insert(CanonicalIVPHI, Header->begin());
 
-  VPBuilder Builder(TopRegion->getExitingBasicBlock());
+  VPBuilder Builder(Layout, TopRegion->getExitingBasicBlock());
   // Add a VPInstruction to increment the scalar canonical IV by VF * UF.
   auto *CanonicalIVIncrement = Builder.createOverflowingOp(
       Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()}, {HasNUW, false}, DL,
@@ -9067,15 +9069,16 @@ static VPInstruction *addResumePhiRecipeForInduction(
 /// original phis in the scalar header. End values for inductions are added to
 /// \p IVEndValues.
 static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
-                                DenseMap<VPValue *, VPValue *> &IVEndValues) {
+                                DenseMap<VPValue *, VPValue *> &IVEndValues,
+                                const DataLayout &DL) {
   VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
   auto *ScalarPH = Plan.getScalarPreheader();
   auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getSinglePredecessor());
   VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
   VPBuilder VectorPHBuilder(
-      cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
-  VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
-  VPBuilder ScalarPHBuilder(ScalarPH);
+      DL, cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
+  VPBuilder MiddleBuilder(DL, MiddleVPBB, MiddleVPBB->getFirstNonPhi());
+  VPBuilder ScalarPHBuilder(DL, ScalarPH);
   VPValue *OneVPV = Plan.getOrAddLiveIn(
       ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
   for (VPRecipeBase &ScalarPhiR : *Plan.getScalarHeader()) {
@@ -9165,12 +9168,13 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
 // ExitUsersToFix if needed and their operands are updated.
 static void
 addUsersInExitBlocks(VPlan &Plan,
-                     const SetVector<VPIRInstruction *> &ExitUsersToFix) {
+                     const SetVector<VPIRInstruction *> &ExitUsersToFix,
+                     const DataLayout &DL) {
   if (ExitUsersToFix.empty())
     return;
 
   auto *MiddleVPBB = Plan.getMiddleBlock();
-  VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
+  VPBuilder B(DL, MiddleVPBB, MiddleVPBB->getFirstNonPhi());
 
   // Introduce extract for exiting values and update the VPIRInstructions
   // modeling the corresponding LCSSA phis.
@@ -9188,12 +9192,13 @@ addUsersInExitBlocks(VPlan &Plan,
 /// users in the original exit block using the VPIRInstruction wrapping to the
 /// LCSSA phi.
 static void addExitUsersForFirstOrderRecurrences(
-    VPlan &Plan, SetVector<VPIRInstruction *> &ExitUsersToFix) {
+    VPlan &Plan, SetVector<VPIRInstruction *> &ExitUsersToFix,
+    const DataLayout &DL) {
   VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
   auto *ScalarPHVPBB = Plan.getScalarPreheader();
   auto *MiddleVPBB = Plan.getMiddleBlock();
-  VPBuilder ScalarPHBuilder(ScalarPHVPBB);
-  VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
+  VPBuilder ScalarPHBuilder(DL, ScalarPHVPBB);
+  VPBuilder MiddleBuilder(DL, MiddleVPBB, MiddleVPBB->getFirstNonPhi());
   VPValue *TwoVPV = Plan.getOrAddLiveIn(
       ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 2));
 
@@ -9336,7 +9341,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
   // that the canonical induction increment will not overflow as the vector trip
   // count is >= increment and a multiple of the increment.
   bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
-  addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW, DL);
+  const DataLayout &Layout = OrigLoop->getHeader()->getDataLayout();
+  addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW, DL,
+                        Layout);
 
   VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
                                 Builder);
@@ -9523,11 +9530,11 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
                              RecipeBuilder);
   }
   DenseMap<VPValue *, VPValue *> IVEndValues;
-  addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
+  addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues, Layout);
   SetVector<VPIRInstruction *> ExitUsersToFix =
       collectUsersInExitBlocks(OrigLoop, RecipeBuilder, *Plan);
-  addExitUsersForFirstOrderRecurrences(*Plan, ExitUsersToFix);
-  addUsersInExitBlocks(*Plan, ExitUsersToFix);
+  addExitUsersForFirstOrderRecurrences(*Plan, ExitUsersToFix, Layout);
+  addUsersInExitBlocks(*Plan, ExitUsersToFix, Layout);
 
   // ---------------------------------------------------------------------------
   // Transform initial VPlan: Apply previously taken decisions, in order, to
@@ -9599,7 +9606,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
     bool WithoutRuntimeCheck =
         Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
     VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow,
-                                       WithoutRuntimeCheck);
+                                       WithoutRuntimeCheck, Layout);
   }
   VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues);
 
@@ -9638,8 +9645,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
   // Tail folding is not supported for outer loops, so the induction increment
   // is guaranteed to not wrap.
   bool HasNUW = true;
+  const DataLayout &DL = OrigLoop->getHeader()->getDataLayout();
   addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW,
-                        DebugLoc());
+                        DebugLoc(), DL);
 
   // Collect mapping of IR header phis to header phi recipes, to be used in
   // addScalarResumePhis.
@@ -9654,7 +9662,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
   DenseMap<VPValue *, VPValue *> IVEndValues;
   // TODO: IVEndValues are not used yet in the native path, to optimize exit
   // values.
-  addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
+  addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues, DL);
 
   assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
   return Plan;
@@ -10105,7 +10113,7 @@ static bool processLoopInVPlanNativePath(
   // TODO: CM is not used at this point inside the planner. Turn CM into an
   // optional argument if we don't need it in the future.
   LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, LVL, CM, IAI, PSE, Hints,
-                               ORE);
+                               ORE, F->getDataLayout());
 
   // Get user vectorization factor.
   ElementCount UserVF = Hints.getWidth();
@@ -10333,7 +10341,8 @@ LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
 /// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
 /// vectorization. Remove ResumePhis from \p MainPlan for inductions that
 /// don't have a corresponding wide induction in \p EpiPlan.
-static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
+static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan,
+                                         const DataLayout &DL) {
   // Collect PHI nodes of widened phis in the VPlan for the epilogue. Those
   // will need their resume-values computed in the main vector loop. Others
   // can be removed from the main VPlan.
@@ -10372,7 +10381,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
                              m_Specific(VectorTC), m_SpecificInt(0)));
       }))
     return;
-  VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
+  VPBuilder ScalarPHBuilder(DL, MainScalarPH, MainScalarPH->begin());
   ScalarPHBuilder.createNaryOp(
       VPInstruction::ResumePhi,
       {VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {},
@@ -10694,7 +10703,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
                                 F, &Hints, IAI, PSI, BFI);
   // Use the planner for vectorization.
   LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
-                               ORE);
+                               ORE, F->getDataLayout());
 
   // Get user vectorization factor and interleave count.
   ElementCount UserVF = Hints.getWidth();
@@ -10877,7 +10886,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         // factor) again shortly afterwards.
         VPlan &BestEpiPlan = LVP.getPlanFor(EpilogueVF.Width);
         BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
-        preparePlanForMainVectorLoop(*BestMainPlan, BestEpiPlan);
+        preparePlanForMainVectorLoop(*BestMainPlan, BestEpiPlan,
+                                     F->getDataLayout());
         EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1,
                                           BestEpiPlan);
         EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TLI, TTI, AC, ORE,

llvm/lib/Transforms/Vectorize/VPlanConstantFolder.h

@@ -7,15 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "VPlanValue.h"
-#include "llvm/IR/ConstantFolder.h"
+#include "llvm/Analysis/TargetFolder.h"
 
 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANCONSTANTFOLDER_H
 #define LLVM_TRANSFORMS_VECTORIZE_VPLANCONSTANTFOLDER_H
 
 namespace llvm {
 class VPConstantFolder {
 private:
-  ConstantFolder Folder;
+  TargetFolder Folder;
 
   Constant *getIRConstant(VPValue *V) const {
     if (!V->isLiveIn())
@@ -33,6 +33,8 @@ class VPConstantFolder {
   }
 
 public:
+  VPConstantFolder(const DataLayout &DL) : Folder(DL) {}
+
   Value *foldAnd(VPValue *LHS, VPValue *RHS) const {
     return foldBinOp(Instruction::BinaryOps::And, LHS, RHS);
   }

llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp

@@ -88,7 +88,7 @@ void VPlanTransforms::introduceTopLevelVectorLoopRegion(
   // of the corresponding compare because they may have ended up with
   // different line numbers and we want to avoid awkward line stepping while
   // debugging. Eg. if the compare has got a line number inside the loop.
-  VPBuilder Builder(MiddleVPBB);
+  VPBuilder Builder(TheLoop->getHeader()->getDataLayout(), MiddleVPBB);
   VPValue *Cmp =
       TailFolded
           ? Plan.getOrAddLiveIn(ConstantInt::getTrue(

llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp

@@ -73,7 +73,8 @@ class PlainCFGBuilder {
 
 public:
   PlainCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
-      : TheLoop(Lp), LI(LI), Plan(P) {}
+      : TheLoop(Lp), LI(LI), Plan(P),
+        VPIRBuilder(Lp->getHeader()->getDataLayout()) {}
 
   /// Build plain CFG for TheLoop  and connects it to Plan's entry.
   void buildPlainCFG(DenseMap<VPBlockBase *, BasicBlock *> &VPB2IRBB);