Skip to content

[SLP]Do extra analysis int minbitwidth if some checks return false. #84363

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 14, 2024
Merged

[SLP]Do extra analysis int minbitwidth if some checks return false. #84363

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d50a0a1
[𝘀𝗽𝗿] initial version
alexey-bataev Mar 7, 2024
a9760d8
Rebase
alexey-bataev Mar 8, 2024
7fe9334
Rebase
alexey-bataev Mar 14, 2024
cabfac6
Rebase, aggregate some common code
alexey-bataev Mar 14, 2024
029af5d
Rebase, fix a crash
alexey-bataev Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 59 additions & 32 deletions llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10225,9 +10225,11 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
for (const TreeEntry *TE : ForRemoval)
Set.erase(TE);
}
bool NeedToRemapValues = false;
for (auto *It = UsedTEs.begin(); It != UsedTEs.end();) {
if (It->empty()) {
UsedTEs.erase(It);
NeedToRemapValues = true;
continue;
}
std::advance(It, 1);
Expand All @@ -10236,6 +10238,19 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
Entries.clear();
return std::nullopt;
}
// Recalculate the mapping between the values and entries sets.
if (NeedToRemapValues) {
DenseMap<Value *, int> PrevUsedValuesEntry;
PrevUsedValuesEntry.swap(UsedValuesEntry);
for (auto [Idx, Set] : enumerate(UsedTEs)) {
DenseSet<Value *> Values;
for (const TreeEntry *E : Set)
Values.insert(E->Scalars.begin(), E->Scalars.end());
for (const auto &P : PrevUsedValuesEntry)
if (Values.contains(P.first))
UsedValuesEntry.try_emplace(P.first, Idx);
}
}
}

unsigned VF = 0;
Expand Down Expand Up @@ -14001,6 +14016,33 @@ bool BoUpSLP::collectValuesToDemote(
};
unsigned Start = 0;
unsigned End = I->getNumOperands();

auto FinalAnalysis = [&](const TreeEntry *ITE = nullptr) {
if (!IsProfitableToDemote)
return false;
return (ITE && ITE->UserTreeIndices.size() > 1) ||
IsPotentiallyTruncated(I, BitWidth);
};
auto ProcessOperands = [&](ArrayRef<Value *> Operands, bool &NeedToExit) {
NeedToExit = false;
unsigned InitLevel = MaxDepthLevel;
for (Value *IncValue : Operands) {
unsigned Level = InitLevel;
if (!collectValuesToDemote(IncValue, IsProfitableToDemoteRoot, BitWidth,
ToDemote, DemotedConsts, Visited, Level,
IsProfitableToDemote, IsTruncRoot)) {
if (!IsProfitableToDemote)
return false;
NeedToExit = true;
if (!FinalAnalysis(ITE))
return false;
continue;
}
MaxDepthLevel = std::max(MaxDepthLevel, Level);
}
return true;
};
bool NeedToExit = false;
switch (I->getOpcode()) {

// We can always demote truncations and extensions. Since truncations can
Expand All @@ -14026,35 +14068,21 @@ bool BoUpSLP::collectValuesToDemote(
case Instruction::And:
case Instruction::Or:
case Instruction::Xor: {
unsigned Level1, Level2;
if ((ITE->UserTreeIndices.size() > 1 &&
!IsPotentiallyTruncated(I, BitWidth)) ||
!collectValuesToDemote(I->getOperand(0), IsProfitableToDemoteRoot,
BitWidth, ToDemote, DemotedConsts, Visited,
Level1, IsProfitableToDemote, IsTruncRoot) ||
!collectValuesToDemote(I->getOperand(1), IsProfitableToDemoteRoot,
BitWidth, ToDemote, DemotedConsts, Visited,
Level2, IsProfitableToDemote, IsTruncRoot))
if (ITE->UserTreeIndices.size() > 1 && !IsPotentiallyTruncated(I, BitWidth))
return false;
if (!ProcessOperands({I->getOperand(0), I->getOperand(1)}, NeedToExit))
return false;
MaxDepthLevel = std::max(Level1, Level2);
break;
}

// We can demote selects if we can demote their true and false values.
case Instruction::Select: {
if (ITE->UserTreeIndices.size() > 1 && !IsPotentiallyTruncated(I, BitWidth))
return false;
Start = 1;
unsigned Level1, Level2;
SelectInst *SI = cast<SelectInst>(I);
if ((ITE->UserTreeIndices.size() > 1 &&
!IsPotentiallyTruncated(I, BitWidth)) ||
!collectValuesToDemote(SI->getTrueValue(), IsProfitableToDemoteRoot,
BitWidth, ToDemote, DemotedConsts, Visited,
Level1, IsProfitableToDemote, IsTruncRoot) ||
!collectValuesToDemote(SI->getFalseValue(), IsProfitableToDemoteRoot,
BitWidth, ToDemote, DemotedConsts, Visited,
Level2, IsProfitableToDemote, IsTruncRoot))
auto *SI = cast<SelectInst>(I);
if (!ProcessOperands({SI->getTrueValue(), SI->getFalseValue()}, NeedToExit))
return false;
MaxDepthLevel = std::max(Level1, Level2);
break;
}

Expand All @@ -14065,22 +14093,20 @@ bool BoUpSLP::collectValuesToDemote(
MaxDepthLevel = 0;
if (ITE->UserTreeIndices.size() > 1 && !IsPotentiallyTruncated(I, BitWidth))
return false;
for (Value *IncValue : PN->incoming_values()) {
unsigned Level;
if (!collectValuesToDemote(IncValue, IsProfitableToDemoteRoot, BitWidth,
ToDemote, DemotedConsts, Visited, Level,
IsProfitableToDemote, IsTruncRoot))
return false;
MaxDepthLevel = std::max(MaxDepthLevel, Level);
}
SmallVector<Value *> Ops(PN->incoming_values().begin(),
PN->incoming_values().end());
if (!ProcessOperands(Ops, NeedToExit))
return false;
break;
}

// Otherwise, conservatively give up.
default:
MaxDepthLevel = 1;
return IsProfitableToDemote && IsPotentiallyTruncated(I, BitWidth);
return FinalAnalysis();
}
if (NeedToExit)
return true;

++MaxDepthLevel;
// Gather demoted constant operands.
Expand Down Expand Up @@ -14119,15 +14145,17 @@ void BoUpSLP::computeMinimumValueSizes() {

// The first value node for store/insertelement is sext/zext/trunc? Skip it,
// resize to the final type.
bool IsTruncRoot = false;
bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt;
if (NodeIdx != 0 &&
VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
(VectorizableTree[NodeIdx]->getOpcode() == Instruction::ZExt ||
VectorizableTree[NodeIdx]->getOpcode() == Instruction::SExt ||
VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc)) {
assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph.");
++NodeIdx;
IsTruncRoot = VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc;
IsProfitableToDemoteRoot = true;
++NodeIdx;
}

// Analyzed in reduction already and not profitable - exit.
Expand Down Expand Up @@ -14259,7 +14287,6 @@ void BoUpSLP::computeMinimumValueSizes() {
ReductionBitWidth = bit_ceil(ReductionBitWidth);
}
bool IsTopRoot = NodeIdx == 0;
bool IsTruncRoot = false;
while (NodeIdx < VectorizableTree.size() &&
VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) {
Expand Down
20 changes: 11 additions & 9 deletions llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ for.end: ; preds = %for.end.loopexit, %
; YAML-NEXT: Function: test_unrolled_select
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'Vectorized horizontal reduction with cost '
; YAML-NEXT: - Cost: '-36'
; YAML-NEXT: - Cost: '-40'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '10'

Expand All @@ -246,15 +246,17 @@ define i32 @test_unrolled_select(ptr noalias nocapture readonly %blk1, ptr noali
; CHECK-NEXT: [[P2_045:%.*]] = phi ptr [ [[BLK2:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR88:%.*]], [[IF_END_86]] ]
; CHECK-NEXT: [[P1_044:%.*]] = phi ptr [ [[BLK1:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR:%.*]], [[IF_END_86]] ]
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[P1_044]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = zext <8 x i8> [[TMP0]] to <8 x i32>
; CHECK-NEXT: [[TMP1:%.*]] = zext <8 x i8> [[TMP0]] to <8 x i16>
; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr [[P2_045]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i32>
; CHECK-NEXT: [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp slt <8 x i32> [[TMP4]], zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = sub nsw <8 x i32> zeroinitializer, [[TMP4]]
; CHECK-NEXT: [[TMP7:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP6]], <8 x i32> [[TMP4]]
; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP7]])
; CHECK-NEXT: [[OP_RDX]] = add i32 [[TMP8]], [[S_047]]
; CHECK-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i16>
; CHECK-NEXT: [[TMP4:%.*]] = sub <8 x i16> [[TMP1]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = trunc <8 x i16> [[TMP4]] to <8 x i1>
; CHECK-NEXT: [[TMP6:%.*]] = icmp slt <8 x i1> [[TMP5]], zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = sub <8 x i16> zeroinitializer, [[TMP4]]
; CHECK-NEXT: [[TMP8:%.*]] = select <8 x i1> [[TMP6]], <8 x i16> [[TMP7]], <8 x i16> [[TMP4]]
; CHECK-NEXT: [[TMP9:%.*]] = zext <8 x i16> [[TMP8]] to <8 x i32>
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP9]])
; CHECK-NEXT: [[OP_RDX]] = add i32 [[TMP10]], [[S_047]]
; CHECK-NEXT: [[CMP83:%.*]] = icmp slt i32 [[OP_RDX]], [[LIM:%.*]]
; CHECK-NEXT: br i1 [[CMP83]], label [[IF_END_86]], label [[FOR_END_LOOPEXIT:%.*]]
; CHECK: if.end.86:
Expand Down