From abc7c36b844822344aa0e625ce4e96c6144a4e86 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 29 May 2025 20:41:57 +0100 Subject: [PATCH] [VPlan] Manage Sentinel value for FindLastIV in VPlan. Similar to modeling the start value as operand, also model the sentinel value as operand explicitly. This makes all require information for code-gen available directly in VPlan. --- .../Transforms/Vectorize/LoopVectorize.cpp | 20 +++++++++++-------- .../Transforms/Vectorize/VPlanPatternMatch.h | 19 ++++++++++++++++++ .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 ++++---- llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +- .../vplan-printing-reductions.ll | 2 +- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 333e50ee98418..4edc7e04cbe43 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7262,9 +7262,11 @@ static void fixReductionScalarResumeWhenVectorizingEpilog( using namespace llvm::PatternMatch; Value *Cmp, *OrigResumeV, *CmpOp; bool IsExpectedPattern = - match(MainResumeValue, m_Select(m_OneUse(m_Value(Cmp)), - m_Specific(RdxDesc.getSentinelValue()), - m_Value(OrigResumeV))) && + match(MainResumeValue, + m_Select( + m_OneUse(m_Value(Cmp)), + m_Specific(EpiRedResult->getOperand(2)->getLiveInIRValue()), + m_Value(OrigResumeV))) && (match(Cmp, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(OrigResumeV), m_Value(CmpOp))) && ((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp)))); @@ -9242,9 +9244,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (RecurrenceDescriptor::isFindLastIVRecurrenceKind( RdxDesc.getRecurrenceKind())) { VPValue *Start = PhiR->getStartValue(); - FinalReductionResult = - Builder.createNaryOp(VPInstruction::ComputeFindLastIVResult, - {PhiR, Start, NewExitingVPV}, ExitDL); + FinalReductionResult = Builder.createNaryOp( + VPInstruction::ComputeFindLastIVResult, + {PhiR, Start, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()), + NewExitingVPV}, + ExitDL); } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind( RdxDesc.getRecurrenceKind())) { VPValue *Start = PhiR->getStartValue(); @@ -9832,8 +9836,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, BasicBlock *ResumeBB = cast(ResumeV)->getParent(); IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt()); Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]); - ResumeV = - Builder.CreateSelect(Cmp, RdxDesc.getSentinelValue(), ResumeV); + ResumeV = Builder.CreateSelect( + Cmp, RdxResult->getOperand(2)->getLiveInIRValue(), ResumeV); } else { VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV); auto *PhiR = dyn_cast(&R); diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index dfd9fc3d4d719..b2535fe3aa578 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -318,6 +318,25 @@ m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { {Op0, Op1, Op2}); } +template +using Recipe4Op_match = Recipe_match, + Opcode, Commutative, RecipeTys...>; + +template +using VPInstruction4Op_match = + Recipe4Op_match; + +template +inline VPInstruction4Op_match +m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2, + const Op3_t &Op3) { + return VPInstruction4Op_match( + {Op0, Op1, Op2, Op3}); +} template inline UnaryVPInstruction_match m_Freeze(const Op0_t &Op0) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 62b99d98a2b5e..e59ee341763c0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -646,16 +646,16 @@ Value *VPInstruction::generate(VPTransformState &State) { // The recipe's operands are the reduction phi, followed by one operand for // each part of the reduction. - unsigned UF = getNumOperands() - 2; - Value *ReducedPartRdx = State.get(getOperand(2)); + unsigned UF = getNumOperands() - 3; + Value *ReducedPartRdx = State.get(getOperand(3)); for (unsigned Part = 1; Part < UF; ++Part) { ReducedPartRdx = createMinMaxOp(Builder, RecurKind::SMax, ReducedPartRdx, - State.get(getOperand(2 + Part))); + State.get(getOperand(3 + Part))); } return createFindLastIVReduction(Builder, ReducedPartRdx, State.get(getOperand(1), true), - RdxDesc.getSentinelValue()); + getOperand(2)->getLiveInIRValue()); } case VPInstruction::ComputeReductionResult: { // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index e4c068ef175bc..dfb5bfabd22b8 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -347,7 +347,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) { match(&R, m_VPInstruction( m_VPValue(), m_VPValue(Op1))) || match(&R, m_VPInstruction( - m_VPValue(), m_VPValue(), m_VPValue(Op1)))) { + m_VPValue(), m_VPValue(), m_VPValue(), m_VPValue(Op1)))) { addUniformForAllParts(cast(&R)); for (unsigned Part = 1; Part != UF; ++Part) R.addOperand(getValueForPart(Op1, Part)); diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll index 95fbc4260587a..978f1b80d26da 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll @@ -240,7 +240,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<%cond> +; CHECK-NEXT: EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond> ; CHECK-NEXT: EMIT vp<[[EXT:%.+]]> = extract-last-element vp<[[RDX_RES]]> ; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<{{.+}}> ; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>