Skip to content

[AArch64] Expand llvm.histogram intrinsic to support umax, umin, and uadd.sat operations #138447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 11, 2025
Merged

[AArch64] Expand llvm.histogram intrinsic to support umax, umin, and uadd.sat operations #138447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b6275d6
[AArch64] Expand llvm.histogram intrinsic to support umax, umin, and …
RonDahan101 May 4, 2025
79bf7c3
Fix review
RonDahan101 May 8, 2025
c3318cc
Fix comments
RonDahan101 May 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions llvm/docs/LangRef.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20295,6 +20295,9 @@ More update operation types may be added in the future.

declare void @llvm.experimental.vector.histogram.add.v8p0.i32(<8 x ptr> %ptrs, i32 %inc, <8 x i1> %mask)
declare void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> %ptrs, i64 %inc, <vscale x 2 x i1> %mask)
declare void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i32(<8 x ptr> %ptrs, i32 %inc, <8 x i1> %mask)
declare void @llvm.experimental.vector.histogram.umax.v8p0.i32(<8 x ptr> %ptrs, i32 %val, <8 x i1> %mask)
declare void @llvm.experimental.vector.histogram.umin.v8p0.i32(<8 x ptr> %ptrs, i32 %val, <8 x i1> %mask)

Arguments:
""""""""""
Expand Down
18 changes: 18 additions & 0 deletions llvm/include/llvm/IR/Intrinsics.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1947,6 +1947,24 @@ def int_experimental_vector_histogram_add : DefaultAttrsIntrinsic<[],
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], // Mask
[ IntrArgMemOnly ]>;

def int_experimental_vector_histogram_uadd_sat : DefaultAttrsIntrinsic<[],
[ llvm_anyvector_ty, // Vector of pointers
llvm_anyint_ty, // Increment
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], // Mask
[ IntrArgMemOnly ]>;

def int_experimental_vector_histogram_umin : DefaultAttrsIntrinsic<[],
[ llvm_anyvector_ty, // Vector of pointers
llvm_anyint_ty, // Update value
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], // Mask
[ IntrArgMemOnly ]>;

def int_experimental_vector_histogram_umax : DefaultAttrsIntrinsic<[],
[ llvm_anyvector_ty, // Vector of pointers
llvm_anyint_ty, // Update value
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], // Mask
[ IntrArgMemOnly ]>;

// Experimental match
def int_experimental_vector_match : DefaultAttrsIntrinsic<
[ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty> ],
Expand Down
36 changes: 32 additions & 4 deletions llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,29 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,

// FIXME: Do we need to add an alignment parameter to the intrinsic?
unsigned VectorWidth = AddrType->getNumElements();
auto CreateHistogramUpdateValue = [&](IntrinsicInst *CI, Value *Load,
Value *Inc) -> Value * {
Value *UpdateOp;
switch (CI->getIntrinsicID()) {
case Intrinsic::experimental_vector_histogram_add:
UpdateOp = Builder.CreateAdd(Load, Inc);
break;
case Intrinsic::experimental_vector_histogram_uadd_sat:
UpdateOp =
Builder.CreateIntrinsic(Intrinsic::uadd_sat, {EltTy}, {Load, Inc});
break;
case Intrinsic::experimental_vector_histogram_umin:
UpdateOp = Builder.CreateIntrinsic(Intrinsic::umin, {EltTy}, {Load, Inc});
break;
case Intrinsic::experimental_vector_histogram_umax:
UpdateOp = Builder.CreateIntrinsic(Intrinsic::umax, {EltTy}, {Load, Inc});
break;

default:
llvm_unreachable("Unexpected histogram intrinsic");
}
return UpdateOp;
};

// Shorten the way if the mask is a vector of constants.
if (isConstantIntVector(Mask)) {
Expand All @@ -976,8 +999,9 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
continue;
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
Value *Add = Builder.CreateAdd(Load, Inc);
Builder.CreateStore(Add, Ptr);
Value *Update =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
Builder.CreateStore(Update, Ptr);
}
CI->eraseFromParent();
return;
Expand All @@ -997,8 +1021,9 @@ static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI,
Builder.SetInsertPoint(CondBlock->getTerminator());
Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));
Value *Add = Builder.CreateAdd(Load, Inc);
Builder.CreateStore(Add, Ptr);
Value *UpdateOp =
CreateHistogramUpdateValue(cast<IntrinsicInst>(CI), Load, Inc);
Builder.CreateStore(UpdateOp, Ptr);

// Create "else" block, fill it in the next iteration
BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
Expand Down Expand Up @@ -1089,6 +1114,9 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
default:
break;
case Intrinsic::experimental_vector_histogram_add:
case Intrinsic::experimental_vector_histogram_uadd_sat:
case Intrinsic::experimental_vector_histogram_umin:
case Intrinsic::experimental_vector_histogram_umax:
if (TTI.isLegalMaskedVectorHistogram(CI->getArgOperand(0)->getType(),
CI->getArgOperand(1)->getType()))
return false;
Expand Down
Loading