diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 02b79f2053d59..f07ca3d12daa5 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7685,6 +7685,8 @@ DenseMap LoopVectorizationPlanner::executePlan( VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF, OrigLoop->getHeader()->getContext()); VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE); + VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType()); + VPlanTransforms::removeDeadRecipes(BestVPlan); VPlanTransforms::convertToConcreteRecipes(BestVPlan); // Perform the actual loop transformation. diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 1e7c54894243b..9bafc6568bc62 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -170,6 +170,14 @@ struct Recipe_match { if ((!matchRecipeAndOpcode(R) && ...)) return false; + if (!(std::is_same_v || ...) && + isa(R)) { + // Don't match VPWidenEVLRecipe if it is not explicitly part of RecipeTys. + // Otherwise we might match it unexpectedly when trying to match + // VPWidenRecipe, of which VPWidenEVLRecipe is a subclass of. + return false; + } + assert(R->getNumOperands() == std::tuple_size::value && "recipe with matched opcode the expected number of operands"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 7e9ef46133936..cb8bf340423f7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -964,9 +964,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { return R.getVPSingleValue()->replaceAllUsesWith(R.getOperand(1)); } -/// Try to simplify the recipes in \p Plan. Use \p CanonicalIVTy as type for all -/// un-typed live-ins in VPTypeAnalysis. -static void simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) { +void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) { ReversePostOrderTraversal> RPOT( Plan.getEntry()); VPTypeAnalysis TypeInfo(&CanonicalIVTy); @@ -1043,7 +1041,6 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, } Term->eraseFromParent(); - VPlanTransforms::removeDeadRecipes(Plan); Plan.setVF(BestVF); Plan.setUF(BestUF); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 0cd4cf1f22a7d..3dd476a8526d6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -163,6 +163,10 @@ struct VPlanTransforms { /// Lower abstract recipes to concrete ones, that can be codegen'd. static void convertToConcreteRecipes(VPlan &Plan); + /// Perform instcombine-like simplifications on recipes in \p Plan. Use \p + /// CanonicalIVTy as type for all un-typed live-ins in VPTypeAnalysis. + static void simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy); + /// If there's a single exit block, optimize its phi recipes that use exiting /// IV values by feeding them precomputed end values instead, possibly taken /// one step backwards.