Skip to content

Commit ed8ebad

Browse files
lei137lei137
authored
[SelectionDAG] Support integer promotion for VP_LOAD and VP_STORE (#81299)
Add integer promotion support for for VP_LOAD and VP_STORE via legalization of extend and truncate of each form. Patch commandeered from: https://reviews.llvm.org/D109377
1 parent ab4253f commit ed8ebad

File tree

5 files changed

+92
-18
lines changed

5 files changed

+92
-18
lines changed

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
8383
case ISD::EXTRACT_VECTOR_ELT:
8484
Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
8585
case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
86+
case ISD::VP_LOAD:
87+
Res = PromoteIntRes_VP_LOAD(cast<VPLoadSDNode>(N));
88+
break;
8689
case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
8790
break;
8891
case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
@@ -957,6 +960,23 @@ SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
957960
return Res;
958961
}
959962

963+
SDValue DAGTypeLegalizer::PromoteIntRes_VP_LOAD(VPLoadSDNode *N) {
964+
assert(!N->isIndexed() && "Indexed vp_load during type legalization!");
965+
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
966+
ISD::LoadExtType ExtType = (N->getExtensionType() == ISD::NON_EXTLOAD)
967+
? ISD::EXTLOAD
968+
: N->getExtensionType();
969+
SDLoc dl(N);
970+
SDValue Res =
971+
DAG.getLoadVP(N->getAddressingMode(), ExtType, NVT, dl, N->getChain(),
972+
N->getBasePtr(), N->getOffset(), N->getMask(),
973+
N->getVectorLength(), N->getMemoryVT(), N->getMemOperand());
974+
// Legalize the chain result - switch anything that used the old chain to
975+
// use the new one.
976+
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
977+
return Res;
978+
}
979+
960980
SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
961981
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
962982
SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
@@ -1957,6 +1977,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
19571977
case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
19581978
case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
19591979
OpNo); break;
1980+
case ISD::VP_STORE:
1981+
Res = PromoteIntOp_VP_STORE(cast<VPStoreSDNode>(N), OpNo);
1982+
break;
19601983
case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
19611984
OpNo); break;
19621985
case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
@@ -2378,6 +2401,19 @@ SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
23782401
N->getMemoryVT(), N->getMemOperand());
23792402
}
23802403

2404+
SDValue DAGTypeLegalizer::PromoteIntOp_VP_STORE(VPStoreSDNode *N,
2405+
unsigned OpNo) {
2406+
2407+
assert(OpNo == 1 && "Unexpected operand for promotion");
2408+
assert(!N->isIndexed() && "expecting unindexed vp_store!");
2409+
2410+
SDValue DataOp = GetPromotedInteger(N->getValue());
2411+
return DAG.getTruncStoreVP(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
2412+
N->getMask(), N->getVectorLength(),
2413+
N->getMemoryVT(), N->getMemOperand(),
2414+
N->isCompressingStore());
2415+
}
2416+
23812417
SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
23822418
unsigned OpNo) {
23832419
SDValue DataOp = N->getValue();

‎llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
338338
SDValue PromoteIntRes_FREEZE(SDNode *N);
339339
SDValue PromoteIntRes_INT_EXTEND(SDNode *N);
340340
SDValue PromoteIntRes_LOAD(LoadSDNode *N);
341+
SDValue PromoteIntRes_VP_LOAD(VPLoadSDNode *N);
341342
SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
342343
SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
343344
SDValue PromoteIntRes_VECTOR_COMPRESS(SDNode *N);
@@ -420,6 +421,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
420421
SDValue PromoteIntOp_ExpOp(SDNode *N);
421422
SDValue PromoteIntOp_VECREDUCE(SDNode *N);
422423
SDValue PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo);
424+
SDValue PromoteIntOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo);
423425
SDValue PromoteIntOp_SET_ROUNDING(SDNode *N);
424426
SDValue PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo);
425427
SDValue PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo);

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpstore.ll‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,18 @@ define void @vpstore_v4i8(<4 x i8> %val, ptr %ptr, <4 x i1> %m, i32 zeroext %evl
2828
ret void
2929
}
3030

31+
declare void @llvm.vp.store.v8i7.v8i7.p0(<8 x i7>, ptr, <8 x i1>, i32)
32+
33+
define void @vpstore_v8i7(<8 x i7> %val, ptr %ptr, <8 x i1> %m, i32 zeroext %evl) {
34+
; CHECK-LABEL: vpstore_v8i7:
35+
; CHECK: # %bb.0:
36+
; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma
37+
; CHECK-NEXT: vse8.v v8, (a0), v0.t
38+
; CHECK-NEXT: ret
39+
call void @llvm.vp.store.v8i7.v8i7.p0(<8 x i7> %val, ptr %ptr, <8 x i1> %m, i32 %evl)
40+
ret void
41+
}
42+
3143
declare void @llvm.vp.store.v8i8.p0(<8 x i8>, ptr, <8 x i1>, i32)
3244

3345
define void @vpstore_v8i8(<8 x i8> %val, ptr %ptr, <8 x i1> %m, i32 zeroext %evl) {
@@ -285,10 +297,10 @@ define void @vpstore_v32f64(<32 x double> %val, ptr %ptr, <32 x i1> %m, i32 zero
285297
; CHECK: # %bb.0:
286298
; CHECK-NEXT: li a3, 16
287299
; CHECK-NEXT: mv a2, a1
288-
; CHECK-NEXT: bltu a1, a3, .LBB23_2
300+
; CHECK-NEXT: bltu a1, a3, .LBB24_2
289301
; CHECK-NEXT: # %bb.1:
290302
; CHECK-NEXT: li a2, 16
291-
; CHECK-NEXT: .LBB23_2:
303+
; CHECK-NEXT: .LBB24_2:
292304
; CHECK-NEXT: vsetvli zero, a2, e64, m8, ta, ma
293305
; CHECK-NEXT: vse64.v v8, (a0), v0.t
294306
; CHECK-NEXT: addi a2, a1, -16

‎llvm/test/CodeGen/RISCV/rvv/vpload.ll‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,18 @@ define <vscale x 3 x i8> @vpload_nxv3i8(ptr %ptr, <vscale x 3 x i1> %m, i32 zero
6565
ret <vscale x 3 x i8> %load
6666
}
6767

68+
declare <vscale x 4 x i6> @llvm.vp.load.nxv4i6.nxv4i6.p0(<vscale x 4 x i6>*, <vscale x 4 x i1>, i32)
69+
70+
define <vscale x 4 x i6> @vpload_nxv4i6(<vscale x 4 x i6>* %ptr, <vscale x 4 x i1> %m, i32 zeroext %evl) {
71+
; CHECK-LABEL: vpload_nxv4i6:
72+
; CHECK: # %bb.0:
73+
; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma
74+
; CHECK-NEXT: vle8.v v8, (a0), v0.t
75+
; CHECK-NEXT: ret
76+
%load = call <vscale x 4 x i6> @llvm.vp.load.nxv4i6.nxv4i6.p0(<vscale x 4 x i6>* %ptr, <vscale x 4 x i1> %m, i32 %evl)
77+
ret <vscale x 4 x i6> %load
78+
}
79+
6880
declare <vscale x 4 x i8> @llvm.vp.load.nxv4i8.p0(ptr, <vscale x 4 x i1>, i32)
6981

7082
define <vscale x 4 x i8> @vpload_nxv4i8(ptr %ptr, <vscale x 4 x i1> %m, i32 zeroext %evl) {
@@ -523,10 +535,10 @@ define <vscale x 16 x double> @vpload_nxv16f64(ptr %ptr, <vscale x 16 x i1> %m,
523535
; CHECK-NEXT: add a4, a0, a4
524536
; CHECK-NEXT: vsetvli zero, a3, e64, m8, ta, ma
525537
; CHECK-NEXT: vle64.v v16, (a4), v0.t
526-
; CHECK-NEXT: bltu a1, a2, .LBB43_2
538+
; CHECK-NEXT: bltu a1, a2, .LBB44_2
527539
; CHECK-NEXT: # %bb.1:
528540
; CHECK-NEXT: mv a1, a2
529-
; CHECK-NEXT: .LBB43_2:
541+
; CHECK-NEXT: .LBB44_2:
530542
; CHECK-NEXT: vmv1r.v v0, v8
531543
; CHECK-NEXT: vsetvli zero, a1, e64, m8, ta, ma
532544
; CHECK-NEXT: vle64.v v8, (a0), v0.t
@@ -553,10 +565,10 @@ define <vscale x 16 x double> @vpload_nxv17f64(ptr %ptr, ptr %out, <vscale x 17
553565
; CHECK-NEXT: csrr a3, vlenb
554566
; CHECK-NEXT: slli a5, a3, 1
555567
; CHECK-NEXT: mv a4, a2
556-
; CHECK-NEXT: bltu a2, a5, .LBB44_2
568+
; CHECK-NEXT: bltu a2, a5, .LBB45_2
557569
; CHECK-NEXT: # %bb.1:
558570
; CHECK-NEXT: mv a4, a5
559-
; CHECK-NEXT: .LBB44_2:
571+
; CHECK-NEXT: .LBB45_2:
560572
; CHECK-NEXT: sub a6, a4, a3
561573
; CHECK-NEXT: slli a7, a3, 3
562574
; CHECK-NEXT: srli t0, a3, 3
@@ -572,21 +584,21 @@ define <vscale x 16 x double> @vpload_nxv17f64(ptr %ptr, ptr %out, <vscale x 17
572584
; CHECK-NEXT: sltu a2, a2, a5
573585
; CHECK-NEXT: addi a2, a2, -1
574586
; CHECK-NEXT: and a2, a2, a5
575-
; CHECK-NEXT: bltu a2, a3, .LBB44_4
587+
; CHECK-NEXT: bltu a2, a3, .LBB45_4
576588
; CHECK-NEXT: # %bb.3:
577589
; CHECK-NEXT: mv a2, a3
578-
; CHECK-NEXT: .LBB44_4:
590+
; CHECK-NEXT: .LBB45_4:
579591
; CHECK-NEXT: slli a5, a3, 4
580592
; CHECK-NEXT: srli a6, a3, 2
581593
; CHECK-NEXT: vsetvli a7, zero, e8, mf2, ta, ma
582594
; CHECK-NEXT: vslidedown.vx v0, v8, a6
583595
; CHECK-NEXT: add a5, a0, a5
584596
; CHECK-NEXT: vsetvli zero, a2, e64, m8, ta, ma
585597
; CHECK-NEXT: vle64.v v24, (a5), v0.t
586-
; CHECK-NEXT: bltu a4, a3, .LBB44_6
598+
; CHECK-NEXT: bltu a4, a3, .LBB45_6
587599
; CHECK-NEXT: # %bb.5:
588600
; CHECK-NEXT: mv a4, a3
589-
; CHECK-NEXT: .LBB44_6:
601+
; CHECK-NEXT: .LBB45_6:
590602
; CHECK-NEXT: vmv1r.v v0, v8
591603
; CHECK-NEXT: vsetvli zero, a4, e64, m8, ta, ma
592604
; CHECK-NEXT: vle64.v v8, (a0), v0.t

‎llvm/test/CodeGen/RISCV/rvv/vpstore.ll‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,18 @@ define void @vpstore_nxv4i16(<vscale x 4 x i16> %val, ptr %ptr, <vscale x 4 x i1
104104
ret void
105105
}
106106

107+
declare void @llvm.vp.store.nxv8i12.nxv8i12.p0(<vscale x 8 x i12>, <vscale x 8 x i12>*, <vscale x 8 x i1>, i32)
108+
109+
define void @vpstore_nxv8i12(<vscale x 8 x i12> %val, <vscale x 8 x i12>* %ptr, <vscale x 8 x i1> %m, i32 zeroext %evl) {
110+
; CHECK-LABEL: vpstore_nxv8i12:
111+
; CHECK: # %bb.0:
112+
; CHECK-NEXT: vsetvli zero, a1, e16, m2, ta, ma
113+
; CHECK-NEXT: vse16.v v8, (a0), v0.t
114+
; CHECK-NEXT: ret
115+
call void @llvm.vp.store.nxv8i12.nxv8i12.p0(<vscale x 8 x i12> %val, <vscale x 8 x i12>* %ptr, <vscale x 8 x i1> %m, i32 %evl)
116+
ret void
117+
}
118+
107119
declare void @llvm.vp.store.nxv8i16.p0(<vscale x 8 x i16>, ptr, <vscale x 8 x i1>, i32)
108120

109121
define void @vpstore_nxv8i16(<vscale x 8 x i16> %val, ptr %ptr, <vscale x 8 x i1> %m, i32 zeroext %evl) {
@@ -421,10 +433,10 @@ define void @vpstore_nxv16f64(<vscale x 16 x double> %val, ptr %ptr, <vscale x 1
421433
; CHECK: # %bb.0:
422434
; CHECK-NEXT: csrr a2, vlenb
423435
; CHECK-NEXT: mv a3, a1
424-
; CHECK-NEXT: bltu a1, a2, .LBB34_2
436+
; CHECK-NEXT: bltu a1, a2, .LBB35_2
425437
; CHECK-NEXT: # %bb.1:
426438
; CHECK-NEXT: mv a3, a2
427-
; CHECK-NEXT: .LBB34_2:
439+
; CHECK-NEXT: .LBB35_2:
428440
; CHECK-NEXT: vsetvli zero, a3, e64, m8, ta, ma
429441
; CHECK-NEXT: vse64.v v8, (a0), v0.t
430442
; CHECK-NEXT: srli a3, a2, 3
@@ -462,15 +474,15 @@ define void @vpstore_nxv17f64(<vscale x 17 x double> %val, ptr %ptr, <vscale x 1
462474
; CHECK-NEXT: csrr a3, vlenb
463475
; CHECK-NEXT: slli a4, a3, 1
464476
; CHECK-NEXT: mv a5, a2
465-
; CHECK-NEXT: bltu a2, a4, .LBB35_2
477+
; CHECK-NEXT: bltu a2, a4, .LBB36_2
466478
; CHECK-NEXT: # %bb.1:
467479
; CHECK-NEXT: mv a5, a4
468-
; CHECK-NEXT: .LBB35_2:
480+
; CHECK-NEXT: .LBB36_2:
469481
; CHECK-NEXT: mv a6, a5
470-
; CHECK-NEXT: bltu a5, a3, .LBB35_4
482+
; CHECK-NEXT: bltu a5, a3, .LBB36_4
471483
; CHECK-NEXT: # %bb.3:
472484
; CHECK-NEXT: mv a6, a3
473-
; CHECK-NEXT: .LBB35_4:
485+
; CHECK-NEXT: .LBB36_4:
474486
; CHECK-NEXT: vmv1r.v v0, v24
475487
; CHECK-NEXT: vl8re64.v v16, (a0)
476488
; CHECK-NEXT: vsetvli zero, a6, e64, m8, ta, ma
@@ -492,10 +504,10 @@ define void @vpstore_nxv17f64(<vscale x 17 x double> %val, ptr %ptr, <vscale x 1
492504
; CHECK-NEXT: vl8r.v v8, (a2) # Unknown-size Folded Reload
493505
; CHECK-NEXT: vsetvli zero, a5, e64, m8, ta, ma
494506
; CHECK-NEXT: vse64.v v8, (a6), v0.t
495-
; CHECK-NEXT: bltu a0, a3, .LBB35_6
507+
; CHECK-NEXT: bltu a0, a3, .LBB36_6
496508
; CHECK-NEXT: # %bb.5:
497509
; CHECK-NEXT: mv a0, a3
498-
; CHECK-NEXT: .LBB35_6:
510+
; CHECK-NEXT: .LBB36_6:
499511
; CHECK-NEXT: slli a2, a3, 4
500512
; CHECK-NEXT: srli a3, a3, 2
501513
; CHECK-NEXT: vsetvli a4, zero, e8, mf2, ta, ma

0 commit comments

Comments
 (0)