Skip to content

[X86][Clang] Add AVX512 Integer Comparison Intrinsics for constexpr Evaluation #164026

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Oct 28, 2025
+248 −30
Merged

[X86][Clang] Add AVX512 Integer Comparison Intrinsics for constexpr Evaluation #164026

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
dbcb924
[X86][Clang] VectorExprEvaluator::VisitCallExpr / InterpretBuiltin - …
sskzakaria Oct 17, 2025
fec68e4
build error
sskzakaria Oct 18, 2025
ee703e0
fixed argument order
sskzakaria Oct 18, 2025
3b6e186
moved the type switch into the loop body
sskzakaria Oct 18, 2025
c3cdc21
format
sskzakaria Oct 18, 2025
151e23b
removing repeated elem(ElemNum).toAPSInt() calls to top of the loop
sskzakaria Oct 18, 2025
e96b79b
Apply clang-format
sskzakaria Oct 18, 2025
4895fd9
Revert "Apply clang-format"
sskzakaria Oct 20, 2025
b8b4c35
Reducing size of TYPE_SWITCH macro expansion
sskzakaria Oct 20, 2025
69ca76f
changed constexpr tests to used varied values
sskzakaria Oct 20, 2025
d84d68c
Reduce type switch size in interp__builtin_cmp_mask
sskzakaria Oct 22, 2025
5aeb4e4
minor changes
sskzakaria Oct 24, 2025
f65d516
clang format
sskzakaria Oct 24, 2025
63a8621
Merge branch 'main' into constexpr_cmp
sskzakaria Oct 24, 2025
a7b7aa7
Merge branch 'main' into constexpr_cmp
sskzakaria Oct 24, 2025
827aee4
Update clang/lib/AST/ByteCode/InterpBuiltin.cpp
sskzakaria Oct 27, 2025
897d74d
Update clang/lib/AST/ExprConstant.cpp
sskzakaria Oct 27, 2025
897b66a
clang format
sskzakaria Oct 27, 2025
c71415a
Update clang/lib/AST/ExprConstant.cpp
sskzakaria Oct 28, 2025
ee5c46c
Merge branch 'main' into constexpr_cmp
sskzakaria Oct 28, 2025
c5c2ef4
Merge branch 'main' into constexpr_cmp
RKSimon Oct 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 36 additions & 18 deletions clang/include/clang/Basic/BuiltinsX86.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1272,81 +1272,99 @@ let Features = "avx512bw", Attributes = [NoThrow, Const, Constexpr] in {
def knotdi : X86Builtin<"unsigned long long int(unsigned long long int)">;
}

let Features = "avx512vl,avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
let Features = "avx512vl,avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
def cmpb128_mask : X86Builtin<"unsigned short(_Vector<16, char>, _Vector<16, char>, _Constant int, unsigned short)">;
}

let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
let Features = "avx512vl",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
def cmpd128_mask : X86Builtin<"unsigned char(_Vector<4, int>, _Vector<4, int>, _Constant int, unsigned char)">;
def cmpq128_mask : X86Builtin<"unsigned char(_Vector<2, long long int>, _Vector<2, long long int>, _Constant int, unsigned char)">;
}

let Features = "avx512vl,avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
let Features = "avx512vl,avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
def cmpw128_mask : X86Builtin<"unsigned char(_Vector<8, short>, _Vector<8, short>, _Constant int, unsigned char)">;
}

let Features = "avx512vl,avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
let Features = "avx512vl,avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
def cmpb256_mask : X86Builtin<"unsigned int(_Vector<32, char>, _Vector<32, char>, _Constant int, unsigned int)">;
}

let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
let Features = "avx512vl",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
def cmpd256_mask : X86Builtin<"unsigned char(_Vector<8, int>, _Vector<8, int>, _Constant int, unsigned char)">;
def cmpq256_mask : X86Builtin<"unsigned char(_Vector<4, long long int>, _Vector<4, long long int>, _Constant int, unsigned char)">;
}

let Features = "avx512vl,avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
let Features = "avx512vl,avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
def cmpw256_mask : X86Builtin<"unsigned short(_Vector<16, short>, _Vector<16, short>, _Constant int, unsigned short)">;
}

let Features = "avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
let Features = "avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
def cmpb512_mask : X86Builtin<"unsigned long long int(_Vector<64, char>, _Vector<64, char>, _Constant int, unsigned long long int)">;
}

let Features = "avx512f", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
let Features = "avx512f",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
def cmpd512_mask : X86Builtin<"unsigned short(_Vector<16, int>, _Vector<16, int>, _Constant int, unsigned short)">;
def cmpq512_mask : X86Builtin<"unsigned char(_Vector<8, long long int>, _Vector<8, long long int>, _Constant int, unsigned char)">;
}

let Features = "avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
let Features = "avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
def cmpw512_mask : X86Builtin<"unsigned int(_Vector<32, short>, _Vector<32, short>, _Constant int, unsigned int)">;
}

let Features = "avx512vl,avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
let Features = "avx512vl,avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
def ucmpb128_mask : X86Builtin<"unsigned short(_Vector<16, char>, _Vector<16, char>, _Constant int, unsigned short)">;
}

let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
let Features = "avx512vl",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
def ucmpd128_mask : X86Builtin<"unsigned char(_Vector<4, int>, _Vector<4, int>, _Constant int, unsigned char)">;
def ucmpq128_mask : X86Builtin<"unsigned char(_Vector<2, long long int>, _Vector<2, long long int>, _Constant int, unsigned char)">;
}

let Features = "avx512vl,avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
let Features = "avx512vl,avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
def ucmpw128_mask : X86Builtin<"unsigned char(_Vector<8, short>, _Vector<8, short>, _Constant int, unsigned char)">;
}

let Features = "avx512vl,avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
let Features = "avx512vl,avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
def ucmpb256_mask : X86Builtin<"unsigned int(_Vector<32, char>, _Vector<32, char>, _Constant int, unsigned int)">;
}

let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
let Features = "avx512vl",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
def ucmpd256_mask : X86Builtin<"unsigned char(_Vector<8, int>, _Vector<8, int>, _Constant int, unsigned char)">;
def ucmpq256_mask : X86Builtin<"unsigned char(_Vector<4, long long int>, _Vector<4, long long int>, _Constant int, unsigned char)">;
}

let Features = "avx512vl,avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
let Features = "avx512vl,avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
def ucmpw256_mask : X86Builtin<"unsigned short(_Vector<16, short>, _Vector<16, short>, _Constant int, unsigned short)">;
}

let Features = "avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
let Features = "avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
def ucmpb512_mask : X86Builtin<"unsigned long long int(_Vector<64, char>, _Vector<64, char>, _Constant int, unsigned long long int)">;
}

let Features = "avx512f", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
let Features = "avx512f",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
def ucmpd512_mask : X86Builtin<"unsigned short(_Vector<16, int>, _Vector<16, int>, _Constant int, unsigned short)">;
def ucmpq512_mask : X86Builtin<"unsigned char(_Vector<8, long long int>, _Vector<8, long long int>, _Constant int, unsigned char)">;
}

let Features = "avx512bw", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
let Features = "avx512bw",
Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
def ucmpw512_mask : X86Builtin<"unsigned int(_Vector<32, short>, _Vector<32, short>, _Constant int, unsigned int)">;
}

Expand Down
85 changes: 85 additions & 0 deletions clang/lib/AST/ByteCode/InterpBuiltin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3101,6 +3101,61 @@ static bool interp__builtin_vec_set(InterpState &S, CodePtr OpPC,
return true;
}

static bool evalICmpImm(const uint8_t imm, APSInt A, APSInt B,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(style) imm -> Imm (drop const)

bool IsUnsigned) {
switch (imm & 0x7) {
case 0x00: // _MM_CMPINT_EQ
return (A == B);
case 0x01: // _MM_CMPINT_LT
return IsUnsigned ? A.ult(B) : A.slt(B);
case 0x02: // _MM_CMPINT_LE
return IsUnsigned ? A.ule(B) : A.sle(B);
case 0x03: // _MM_CMPINT_FALSE
return false;
case 0x04: // _MM_CMPINT_NE
return (A != B);
case 0x05: // _MM_CMPINT_NLT
return IsUnsigned ? A.ugt(B) : A.sgt(B);
case 0x06: // _MM_CMPINT_NLE
return IsUnsigned ? A.uge(B) : A.sge(B);
case 0x07: // _MM_CMPINT_TRUE
return true;
default:
llvm_unreachable("Invalid Op");
}
}

static bool interp__builtin_cmp_mask(InterpState &S, CodePtr OpPC,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

interp__builtin_cmp_mask -> interp__builtin_ia32_cmp_mask

const CallExpr *Call, unsigned ID,
bool IsUnsigned) {
assert(Call->getNumArgs() == 4);

APSInt Mask = popToAPSInt(S, Call->getArg(3));
APSInt Opcode = popToAPSInt(S, Call->getArg(2));
unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();

assert(LHS.getNumElems() == RHS.getNumElems());

APInt RetMask = APInt::getZero(LHS.getNumElems());
unsigned VectorLen = LHS.getNumElems();
PrimType ElemT = LHS.getFieldDesc()->getPrimType();

for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
APSInt A;
APSInt B;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(style) APSInt A, B;

INT_TYPE_SWITCH_NO_BOOL(ElemT, {
A = LHS.elem<T>(ElemNum).toAPSInt();
B = RHS.elem<T>(ElemNum).toAPSInt();
});
RetMask.setBitVal(ElemNum,
Mask[ElemNum] && evalICmpImm(CmpOp, A, B, IsUnsigned));
}
pushInteger(S, RetMask, Call->getType());
return true;
}

static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
assert(Call->getNumArgs() == 1);
Expand Down Expand Up @@ -4141,6 +4196,36 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
case X86::BI__builtin_ia32_vec_set_v4di:
return interp__builtin_vec_set(S, OpPC, Call, BuiltinID);

case X86::BI__builtin_ia32_cmpb128_mask:
case X86::BI__builtin_ia32_cmpw128_mask:
case X86::BI__builtin_ia32_cmpd128_mask:
case X86::BI__builtin_ia32_cmpq128_mask:
case X86::BI__builtin_ia32_cmpb256_mask:
case X86::BI__builtin_ia32_cmpw256_mask:
case X86::BI__builtin_ia32_cmpd256_mask:
case X86::BI__builtin_ia32_cmpq256_mask:
case X86::BI__builtin_ia32_cmpb512_mask:
case X86::BI__builtin_ia32_cmpw512_mask:
case X86::BI__builtin_ia32_cmpd512_mask:
case X86::BI__builtin_ia32_cmpq512_mask:
return interp__builtin_cmp_mask(S, OpPC, Call, BuiltinID,
/*IsUnsigned=*/false);

case X86::BI__builtin_ia32_ucmpb128_mask:
case X86::BI__builtin_ia32_ucmpw128_mask:
case X86::BI__builtin_ia32_ucmpd128_mask:
case X86::BI__builtin_ia32_ucmpq128_mask:
case X86::BI__builtin_ia32_ucmpb256_mask:
case X86::BI__builtin_ia32_ucmpw256_mask:
case X86::BI__builtin_ia32_ucmpd256_mask:
case X86::BI__builtin_ia32_ucmpq256_mask:
case X86::BI__builtin_ia32_ucmpb512_mask:
case X86::BI__builtin_ia32_ucmpw512_mask:
case X86::BI__builtin_ia32_ucmpd512_mask:
case X86::BI__builtin_ia32_ucmpq512_mask:
return interp__builtin_cmp_mask(S, OpPC, Call, BuiltinID,
/*IsUnsigned=*/true);

default:
S.FFDiag(S.Current->getLocation(OpPC),
diag::note_invalid_subexpr_in_const_expr)
Expand Down
84 changes: 84 additions & 0 deletions clang/lib/AST/ExprConstant.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15449,6 +15449,90 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
unsigned Idx = static_cast<unsigned>(IdxAPS.getZExtValue() & (N - 1));
return Success(Vec.getVectorElt(Idx).getInt(), E);
}

case clang::X86::BI__builtin_ia32_cmpb128_mask:
case clang::X86::BI__builtin_ia32_cmpw128_mask:
case clang::X86::BI__builtin_ia32_cmpd128_mask:
case clang::X86::BI__builtin_ia32_cmpq128_mask:
case clang::X86::BI__builtin_ia32_cmpb256_mask:
case clang::X86::BI__builtin_ia32_cmpw256_mask:
case clang::X86::BI__builtin_ia32_cmpd256_mask:
case clang::X86::BI__builtin_ia32_cmpq256_mask:
case clang::X86::BI__builtin_ia32_cmpb512_mask:
case clang::X86::BI__builtin_ia32_cmpw512_mask:
case clang::X86::BI__builtin_ia32_cmpd512_mask:
case clang::X86::BI__builtin_ia32_cmpq512_mask:
case clang::X86::BI__builtin_ia32_ucmpb128_mask:
case clang::X86::BI__builtin_ia32_ucmpw128_mask:
case clang::X86::BI__builtin_ia32_ucmpd128_mask:
case clang::X86::BI__builtin_ia32_ucmpq128_mask:
case clang::X86::BI__builtin_ia32_ucmpb256_mask:
case clang::X86::BI__builtin_ia32_ucmpw256_mask:
case clang::X86::BI__builtin_ia32_ucmpd256_mask:
case clang::X86::BI__builtin_ia32_ucmpq256_mask:
case clang::X86::BI__builtin_ia32_ucmpb512_mask:
case clang::X86::BI__builtin_ia32_ucmpw512_mask:
case clang::X86::BI__builtin_ia32_ucmpd512_mask:
case clang::X86::BI__builtin_ia32_ucmpq512_mask: {
assert(E->getNumArgs() == 4);

bool IsUnsigned =
(BuiltinOp >= clang::X86::BI__builtin_ia32_ucmpb128_mask &&
BuiltinOp <= clang::X86::BI__builtin_ia32_ucmpq512_mask);

APValue LHS, RHS;
APSInt Mask, Opcode;
if (!EvaluateVector(E->getArg(0), LHS, Info) ||
!EvaluateVector(E->getArg(1), RHS, Info) ||
!EvaluateInteger(E->getArg(2), Opcode, Info) ||
!EvaluateInteger(E->getArg(3), Mask, Info))
return false;

assert(LHS.getVectorLength() == RHS.getVectorLength());

unsigned VectorLen = LHS.getVectorLength();
unsigned RetWidth = VectorLen ? VectorLen : 1;
if (Mask.getBitWidth() > RetWidth)
RetWidth = Mask.getBitWidth();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How does this happen?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This assertion SI.getBitWidth() == Info.Ctx.getIntWidth(E->getType()) && "Invalid evaluation result. was failing in my original code during the CI build checks so I tried that to see if it would build, but I dont think its needed.


APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);
for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
const APSInt &A = LHS.getVectorElt(ElemNum).getInt();
const APSInt &B = RHS.getVectorElt(ElemNum).getInt();
bool result = false;

switch (Opcode.getExtValue() & 0x7) {
case 0: // _MM_CMPINT_EQ
result = (A == B);
break;
case 1: // _MM_CMPINT_LT
result = IsUnsigned ? A.ult(B) : A.slt(B);
break;
case 2: // _MM_CMPINT_LE
result = IsUnsigned ? A.ule(B) : A.sle(B);
break;
case 3: // _MM_CMPINT_FALSE
result = false;
break;
case 4: // _MM_CMPINT_NE
result = (A != B);
break;
case 5: // _MM_CMPINT_NLT (>=)
result = IsUnsigned ? A.uge(B) : A.sge(B);
break;
case 6: // _MM_CMPINT_NLE (>)
result = IsUnsigned ? A.ugt(B) : A.sgt(B);
break;
case 7: // _MM_CMPINT_TRUE
result = true;
break;
}

RetMask.setBitVal(ElemNum, Mask[ElemNum] && result);
}

return Success(APValue(RetMask), E);
}
}
}

Expand Down
20 changes: 8 additions & 12 deletions clang/lib/Headers/avx512vlbwintrin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2385,22 +2385,19 @@ _mm256_mask_storeu_epi8 (void *__P, __mmask32 __U, __m256i __A)
(__mmask32) __U);
}

static __inline__ __mmask16 __DEFAULT_FN_ATTRS128
_mm_test_epi8_mask (__m128i __A, __m128i __B)
{
static __inline__ __mmask16 __DEFAULT_FN_ATTRS128_CONSTEXPR
_mm_test_epi8_mask(__m128i __A, __m128i __B) {
return _mm_cmpneq_epi8_mask (_mm_and_si128(__A, __B), _mm_setzero_si128());
}

static __inline__ __mmask16 __DEFAULT_FN_ATTRS128
_mm_mask_test_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B)
{
static __inline__ __mmask16 __DEFAULT_FN_ATTRS128_CONSTEXPR
_mm_mask_test_epi8_mask(__mmask16 __U, __m128i __A, __m128i __B) {
return _mm_mask_cmpneq_epi8_mask (__U, _mm_and_si128 (__A, __B),
_mm_setzero_si128());
}

static __inline__ __mmask32 __DEFAULT_FN_ATTRS256
_mm256_test_epi8_mask (__m256i __A, __m256i __B)
{
static __inline__ __mmask32 __DEFAULT_FN_ATTRS256_CONSTEXPR
_mm256_test_epi8_mask(__m256i __A, __m256i __B) {
return _mm256_cmpneq_epi8_mask (_mm256_and_si256(__A, __B),
_mm256_setzero_si256());
}
Expand Down Expand Up @@ -2439,9 +2436,8 @@ _mm256_mask_test_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B)
_mm256_setzero_si256());
}

static __inline__ __mmask16 __DEFAULT_FN_ATTRS128
_mm_testn_epi8_mask (__m128i __A, __m128i __B)
{
static __inline__ __mmask16 __DEFAULT_FN_ATTRS128_CONSTEXPR
_mm_testn_epi8_mask(__m128i __A, __m128i __B) {
return _mm_cmpeq_epi8_mask (_mm_and_si128 (__A, __B), _mm_setzero_si128());
}

Expand Down
Loading