Skip to content

Commit b71515c

Browse files
artagnonartagnon
authored
[VPlan] Extend licm to hoist assumes (#162636)
Assumes are safe to hoist if they're guaranteed to execute, since they don't alias, and don't throw. This mirrors what the IR-LICM does.
1 parent 5614d36 commit b71515c

File tree

5 files changed

+20
-23
lines changed

5 files changed

+20
-23
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2124,6 +2124,11 @@ static void licm(VPlan &Plan) {
21242124
// Return true if we do not know how to (mechanically) hoist a given recipe
21252125
// out of a loop region.
21262126
auto CannotHoistRecipe = [](VPRecipeBase &R) {
2127+
// Assumes don't alias anything or throw; as long as they're guaranteed to
2128+
// execute, they're safe to hoist.
2129+
if (match(&R, m_Intrinsic<Intrinsic::assume>()))
2130+
return false;
2131+
21272132
// TODO: Relax checks in the future, e.g. we could also hoist reads, if
21282133
// their memory location is not modified in the vector loop.
21292134
if (R.mayHaveSideEffects() || R.mayReadFromMemory() || R.isPhi())

‎llvm/test/Transforms/LoopVectorize/X86/cost-model.ll‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -540,6 +540,8 @@ define i64 @cost_assume(ptr %end, i64 %N) {
540540
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 8
541541
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
542542
; CHECK-NEXT: [[TMP11:%.*]] = icmp ne i64 [[N:%.*]], 0
543+
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
544+
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
543545
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
544546
; CHECK: vector.body:
545547
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -551,14 +553,6 @@ define i64 @cost_assume(ptr %end, i64 %N) {
551553
; CHECK-NEXT: [[TMP8]] = add <2 x i64> [[VEC_PHI2]], splat (i64 1)
552554
; CHECK-NEXT: [[TMP9]] = add <2 x i64> [[VEC_PHI3]], splat (i64 1)
553555
; CHECK-NEXT: [[TMP10]] = add <2 x i64> [[VEC_PHI4]], splat (i64 1)
554-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
555-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
556-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
557-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
558-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
559-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
560-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
561-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[TMP11]])
562556
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
563557
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
564558
; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]

‎llvm/test/Transforms/LoopVectorize/assume.ll‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,9 @@ define void @test1(ptr noalias nocapture %a, ptr noalias nocapture readonly %b)
3434
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1600
3535
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
3636
; CHECK: [[MIDDLE_BLOCK]]:
37-
; CHECK-NEXT: br [[FOR_END:label %.*]]
38-
; CHECK: [[SCALAR_PH:.*:]]
37+
; CHECK-NEXT: br label %[[FOR_END:.*]]
38+
; CHECK: [[FOR_END]]:
39+
; CHECK-NEXT: ret void
3940
;
4041
entry:
4142
br label %for.body
@@ -73,29 +74,28 @@ define void @test2(ptr noalias %a, ptr noalias %b) {
7374
; CHECK-NEXT: [[MASKCOND4:%.*]] = icmp eq i64 [[MASKEDPTR3]], 0
7475
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
7576
; CHECK: [[VECTOR_PH]]:
77+
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
78+
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
7679
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
7780
; CHECK: [[VECTOR_BODY]]:
7881
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
79-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
80-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
8182
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
8283
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i32 2
8384
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP3]], align 4
8485
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x float>, ptr [[TMP4]], align 4
8586
; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
8687
; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x float> [[WIDE_LOAD1]], splat (float 1.000000e+00)
87-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
88-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
8988
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
9089
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 2
9190
; CHECK-NEXT: store <2 x float> [[TMP5]], ptr [[TMP7]], align 4
9291
; CHECK-NEXT: store <2 x float> [[TMP6]], ptr [[TMP8]], align 4
9392
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
9493
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1600
95-
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
94+
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
9695
; CHECK: [[MIDDLE_BLOCK]]:
97-
; CHECK-NEXT: br [[FOR_END:label %.*]]
98-
; CHECK: [[SCALAR_PH:.*:]]
96+
; CHECK-NEXT: br label %[[FOR_END:.*]]
97+
; CHECK: [[FOR_END]]:
98+
; CHECK-NEXT: ret void
9999
;
100100
entry:
101101
%ptrint = ptrtoint ptr %a to i64
@@ -163,7 +163,7 @@ define void @predicated_assume(ptr noalias nocapture readonly %a, ptr noalias no
163163
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
164164
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[STEP_ADD]], splat (i64 2)
165165
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
166-
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
166+
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
167167
; CHECK: [[MIDDLE_BLOCK]]:
168168
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
169169
; CHECK-NEXT: br i1 [[CMP_N]], [[FOR_COND_CLEANUP_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]

‎llvm/test/Transforms/LoopVectorize/operand-bundles.ll‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,12 +189,12 @@ define void @assume_cold_operand_bundle(ptr noalias %a, ptr noalias %b) {
189189
; CHECK-NEXT: [[ENTRY:.*:]]
190190
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
191191
; CHECK: [[VECTOR_PH]]:
192+
; CHECK-NEXT: tail call void @llvm.assume(i1 true) [ "cold"() ]
192193
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
193194
; CHECK: [[VECTOR_BODY]]:
194195
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
195196
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
196197
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
197-
; CHECK-NEXT: tail call void @llvm.assume(i1 true) [ "cold"() ]
198198
; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
199199
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
200200
; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[TMP2]], align 4

‎llvm/test/Transforms/LoopVectorize/scalable-assume.ll‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,11 +88,11 @@ define void @test2(ptr %a, ptr noalias %b) {
8888
; CHECK-NEXT: [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
8989
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1600, [[TMP7]]
9090
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1600, [[N_MOD_VF]]
91+
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
92+
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
9193
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
9294
; CHECK: [[VECTOR_BODY]]:
9395
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
94-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
95-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND]])
9696
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
9797
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
9898
; CHECK-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
@@ -101,8 +101,6 @@ define void @test2(ptr %a, ptr noalias %b) {
101101
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <vscale x 2 x float>, ptr [[TMP13]], align 4
102102
; CHECK-NEXT: [[TMP14:%.*]] = fadd <vscale x 2 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
103103
; CHECK-NEXT: [[TMP15:%.*]] = fadd <vscale x 2 x float> [[WIDE_LOAD3]], splat (float 1.000000e+00)
104-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
105-
; CHECK-NEXT: tail call void @llvm.assume(i1 [[MASKCOND4]])
106104
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
107105
; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
108106
; CHECK-NEXT: [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 1

0 commit comments

Comments
 (0)