From 1105b3c38458297ef2a66f36d02024dc01427ed3 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 27 Aug 2025 20:30:40 +0100 Subject: [PATCH 1/2] [ValueTracking] Support GEPs in matchSimpleRecurrence. --- llvm/include/llvm/Analysis/ValueTracking.h | 6 ++- llvm/lib/Analysis/ValueTracking.cpp | 46 +++++++++++++++---- .../InferAlignment/gep-recurrence.ll | 26 +++++------ 3 files changed, 54 insertions(+), 24 deletions(-) diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index 15ff129deda13..562986d9ed458 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -961,7 +961,11 @@ LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step); /// Analogous to the above, but starting from the binary operator -LLVM_ABI bool matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P, +LLVM_ABI bool matchSimpleRecurrence(const Instruction *I, PHINode *&P, + Value *&Start, Value *&Step); + +/// Analogous to the above, but also supporting non-binary operators. +LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, Instruction *&BO, Value *&Start, Value *&Step); /// Attempt to match a simple value-accumulating recurrence of the form: diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 07950f5341d6c..1b71aac463111 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -1577,7 +1577,7 @@ static void computeKnownBitsFromOperator(const Operator *I, } case Instruction::PHI: { const PHINode *P = cast(I); - BinaryOperator *BO = nullptr; + Instruction *BO = nullptr; Value *R = nullptr, *L = nullptr; if (matchSimpleRecurrence(P, BO, R, L)) { // Handle the case of a simple two-predecessor recurrence PHI. @@ -1641,6 +1641,7 @@ static void computeKnownBitsFromOperator(const Operator *I, case Instruction::Sub: case Instruction::And: case Instruction::Or: + case Instruction::GetElementPtr: case Instruction::Mul: { // Change the context instruction to the "edge" that flows into the // phi. This is important because that is where the value is actually @@ -1659,6 +1660,11 @@ static void computeKnownBitsFromOperator(const Operator *I, // We need to take the minimum number of known bits KnownBits Known3(BitWidth); + if (BitWidth != getBitWidth(L->getType(), Q.DL)) { + assert(isa(BO) && + "Bitwidth should only be different for GEPs."); + break; + } RecQ.CxtI = LInst; computeKnownBits(L, DemandedElts, Known3, RecQ, Depth + 1); @@ -1821,6 +1827,7 @@ static void computeKnownBitsFromOperator(const Operator *I, Known.resetAll(); } } + if (const IntrinsicInst *II = dyn_cast(I)) { switch (II->getIntrinsicID()) { default: @@ -2351,7 +2358,7 @@ void computeKnownBits(const Value *V, const APInt &DemandedElts, /// always a power of two (or zero). static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero, SimplifyQuery &Q, unsigned Depth) { - BinaryOperator *BO = nullptr; + Instruction *BO = nullptr; Value *Start = nullptr, *Step = nullptr; if (!matchSimpleRecurrence(PN, BO, Start, Step)) return false; @@ -2389,7 +2396,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero, // Divisor must be a power of two. // If OrZero is false, cannot guarantee induction variable is non-zero after // division, same for Shr, unless it is exact division. - return (OrZero || Q.IIQ.isExact(BO)) && + return (OrZero || Q.IIQ.isExact(cast(BO))) && isKnownToBeAPowerOfTwo(Step, false, Q, Depth); case Instruction::Shl: return OrZero || Q.IIQ.hasNoUnsignedWrap(BO) || Q.IIQ.hasNoSignedWrap(BO); @@ -2398,7 +2405,7 @@ static bool isPowerOfTwoRecurrence(const PHINode *PN, bool OrZero, return false; [[fallthrough]]; case Instruction::LShr: - return OrZero || Q.IIQ.isExact(BO); + return OrZero || Q.IIQ.isExact(cast(BO)); default: return false; } @@ -2810,7 +2817,7 @@ static bool rangeMetadataExcludesValue(const MDNode* Ranges, const APInt& Value) /// Try to detect a recurrence that monotonically increases/decreases from a /// non-zero starting value. These are common as induction variables. static bool isNonZeroRecurrence(const PHINode *PN) { - BinaryOperator *BO = nullptr; + Instruction *BO = nullptr; Value *Start = nullptr, *Step = nullptr; const APInt *StartC, *StepC; if (!matchSimpleRecurrence(PN, BO, Start, Step) || @@ -3648,9 +3655,9 @@ getInvertibleOperands(const Operator *Op1, // If PN1 and PN2 are both recurrences, can we prove the entire recurrences // are a single invertible function of the start values? Note that repeated // application of an invertible function is also invertible - BinaryOperator *BO1 = nullptr; + Instruction *BO1 = nullptr; Value *Start1 = nullptr, *Step1 = nullptr; - BinaryOperator *BO2 = nullptr; + Instruction *BO2 = nullptr; Value *Start2 = nullptr, *Step2 = nullptr; if (PN1->getParent() != PN2->getParent() || !matchSimpleRecurrence(PN1, BO1, Start1, Step1) || @@ -9130,6 +9137,13 @@ static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst, if (LHS != PN && RHS != PN) continue; + Value *LR; + if (match(Operation, m_PtrAdd(m_Specific(PN), m_Value(LR)))) { + Inst = Operation; + Init = PN->getIncomingValue(!I); + OtherOp = LR; + return true; + } Inst = Operation; Init = PN->getIncomingValue(!I); OtherOp = (LHS == PN) ? RHS : LHS; @@ -9147,12 +9161,24 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, // Or: // %iv = [Start, %entry], [%iv.next, %backedge] // %iv.next = binop Step, %iv - return matchTwoInputRecurrence(P, BO, Start, Step); + return matchTwoInputRecurrence(P, BO, Start, Step) && isa(BO); +} + +bool llvm::matchSimpleRecurrence(const PHINode *P, Instruction *&BO, + Value *&Start, Value *&Step) { + // We try to match a recurrence of the form: + // %iv = [Start, %entry], [%iv.next, %backedge] + // %iv.next = binop %iv, Step + // Or: + // %iv = [Start, %entry], [%iv.next, %backedge] + // %iv.next = binop Step, %iv + return matchTwoInputRecurrence(P, BO, Start, Step) && + isa(BO); } -bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P, +bool llvm::matchSimpleRecurrence(const Instruction *I, PHINode *&P, Value *&Start, Value *&Step) { - BinaryOperator *BO = nullptr; + Instruction *BO = nullptr; P = dyn_cast(I->getOperand(0)); if (!P) P = dyn_cast(I->getOperand(1)); diff --git a/llvm/test/Transforms/InferAlignment/gep-recurrence.ll b/llvm/test/Transforms/InferAlignment/gep-recurrence.ll index 50b968d206b83..6211a9ccbc74d 100644 --- a/llvm/test/Transforms/InferAlignment/gep-recurrence.ll +++ b/llvm/test/Transforms/InferAlignment/gep-recurrence.ll @@ -12,7 +12,7 @@ define void @recur_i8_128(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 128 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 128 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -40,7 +40,7 @@ define void @recur_i8_128_no_nusw(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 128 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr i8, ptr [[IV]], i64 128 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -68,7 +68,7 @@ define void @recur_i8_64(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 64 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 64 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -124,7 +124,7 @@ define void @recur_i8_32(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 32 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 32 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -152,7 +152,7 @@ define void @recur_i8_16(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 16 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 16 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -180,7 +180,7 @@ define void @recur_i8_8(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 8 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 8 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -208,7 +208,7 @@ define void @recur_i8_4(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 4 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 4 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -236,7 +236,7 @@ define void @recur_i8_2(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 2 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 2 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -412,7 +412,7 @@ define void @recur_i32_4(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 4 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i32, ptr [[IV]], i64 4 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -440,7 +440,7 @@ define void @recur_i32_3(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 4 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i32, ptr [[IV]], i64 4 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -468,7 +468,7 @@ define void @recur_i8_neg_128(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 128 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -128 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -496,7 +496,7 @@ define void @recur_i8_neg64(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 64 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -64 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] @@ -552,7 +552,7 @@ define void @recur_i8_neg_32(ptr align 128 %dst) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: store i64 0, ptr [[IV]], align 1 +; CHECK-NEXT: store i64 0, ptr [[IV]], align 32 ; CHECK-NEXT: [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -32 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() ; CHECK-NEXT: br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]] From c9626d2ae8e0ee99706f4eb599797efa8b61cc12 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 28 Aug 2025 21:40:56 +0100 Subject: [PATCH 2/2] !fixup move block as suggested, thanks --- llvm/lib/Analysis/ValueTracking.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 1b71aac463111..08a8d723da7fc 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -9132,11 +9132,6 @@ static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst, for (unsigned I = 0; I != 2; ++I) { if (auto *Operation = dyn_cast(PN->getIncomingValue(I)); Operation && Operation->getNumOperands() >= 2) { - Value *LHS = Operation->getOperand(0); - Value *RHS = Operation->getOperand(1); - if (LHS != PN && RHS != PN) - continue; - Value *LR; if (match(Operation, m_PtrAdd(m_Specific(PN), m_Value(LR)))) { Inst = Operation; @@ -9144,6 +9139,12 @@ static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst, OtherOp = LR; return true; } + + Value *LHS = Operation->getOperand(0); + Value *RHS = Operation->getOperand(1); + if (LHS != PN && RHS != PN) + continue; + Inst = Operation; Init = PN->getIncomingValue(!I); OtherOp = (LHS == PN) ? RHS : LHS;