Skip to content

Commit a86509f

Browse files
committed
VPlan: implement VPlan-level constant-folding
Introduce VPlanConstantFolder, a variation of ConstantFolder for VPlan, and use it in VPBuilder to constant-fold when all the underlying IR values passed into the API are constants.
1 parent 4d7f910 commit a86509f

File tree

6 files changed

+287
-68
lines changed

6 files changed

+287
-68
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

Lines changed: 43 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
2626

2727
#include "VPlan.h"
28-
#include "llvm/ADT/SmallSet.h"
28+
#include "VPlanConstantFolder.h"
2929
#include "llvm/Support/InstructionCost.h"
3030

3131
namespace llvm {
@@ -49,6 +49,7 @@ extern cl::opt<unsigned> ForceTargetInstructionCost;
4949
class VPBuilder {
5050
VPBasicBlock *BB = nullptr;
5151
VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();
52+
VPConstantFolder Folder;
5253

5354
/// Insert \p VPI in BB at InsertPt if BB is set.
5455
template <typename T> T *tryInsertInstruction(T *R) {
@@ -69,6 +70,11 @@ class VPBuilder {
6970
return createInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name);
7071
}
7172

73+
VPValue *getOrAddLiveIn(Value *V) {
74+
assert(BB && "Expected insertion point to be set");
75+
return BB->getPlan()->getOrAddLiveIn(V);
76+
}
77+
7278
public:
7379
VPBuilder() = default;
7480
VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); }
@@ -183,31 +189,45 @@ class VPBuilder {
183189

184190
VPValue *createNot(VPValue *Operand, DebugLoc DL = {},
185191
const Twine &Name = "") {
192+
if (auto *V = Folder.foldNot(Operand))
193+
if (BB)
194+
return getOrAddLiveIn(V);
186195
return createInstruction(VPInstruction::Not, {Operand}, DL, Name);
187196
}
188197

189198
VPValue *createAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
190199
const Twine &Name = "") {
200+
if (auto *V = Folder.foldAnd(LHS, RHS))
201+
if (BB)
202+
return getOrAddLiveIn(V);
191203
return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, DL, Name);
192204
}
193205

194206
VPValue *createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
195207
const Twine &Name = "") {
196-
208+
if (auto *V = Folder.foldOr(LHS, RHS))
209+
if (BB)
210+
return getOrAddLiveIn(V);
197211
return tryInsertInstruction(new VPInstruction(
198212
Instruction::BinaryOps::Or, {LHS, RHS},
199213
VPRecipeWithIRFlags::DisjointFlagsTy(false), DL, Name));
200214
}
201215

202216
VPValue *createLogicalAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = {},
203217
const Twine &Name = "") {
218+
if (auto *V = Folder.foldLogicalAnd(LHS, RHS))
219+
if (BB)
220+
return getOrAddLiveIn(V);
204221
return tryInsertInstruction(
205222
new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name));
206223
}
207224

208225
VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
209226
DebugLoc DL = {}, const Twine &Name = "",
210227
std::optional<FastMathFlags> FMFs = std::nullopt) {
228+
if (auto *V = Folder.foldSelect(Cond, TrueVal, FalseVal))
229+
if (BB)
230+
return getOrAddLiveIn(V);
211231
auto *Select =
212232
FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
213233
*FMFs, DL, Name)
@@ -223,17 +243,26 @@ class VPBuilder {
223243
DebugLoc DL = {}, const Twine &Name = "") {
224244
assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&
225245
Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");
246+
if (auto *V = Folder.foldCmp(Pred, A, B))
247+
if (BB)
248+
return getOrAddLiveIn(V);
226249
return tryInsertInstruction(
227250
new VPInstruction(Instruction::ICmp, Pred, A, B, DL, Name));
228251
}
229252

230-
VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = {},
231-
const Twine &Name = "") {
253+
VPValue *createPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = {},
254+
const Twine &Name = "") {
255+
if (auto *V = Folder.foldPtrAdd(Ptr, Offset, GEPNoWrapFlags::none()))
256+
if (BB)
257+
return getOrAddLiveIn(V);
232258
return tryInsertInstruction(
233259
new VPInstruction(Ptr, Offset, GEPNoWrapFlags::none(), DL, Name));
234260
}
235261
VPValue *createInBoundsPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = {},
236262
const Twine &Name = "") {
263+
if (auto *V = Folder.foldPtrAdd(Ptr, Offset, GEPNoWrapFlags::inBounds()))
264+
if (BB)
265+
return getOrAddLiveIn(V);
237266
return tryInsertInstruction(
238267
new VPInstruction(Ptr, Offset, GEPNoWrapFlags::inBounds(), DL, Name));
239268
}
@@ -249,14 +278,20 @@ class VPBuilder {
249278
new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step, Name));
250279
}
251280

252-
VPScalarCastRecipe *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
253-
Type *ResultTy, DebugLoc DL) {
281+
VPValue *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
282+
Type *ResultTy, DebugLoc DL) {
283+
if (auto *V = Folder.foldCast(Opcode, Op, ResultTy))
284+
if (BB)
285+
return getOrAddLiveIn(V);
254286
return tryInsertInstruction(
255287
new VPScalarCastRecipe(Opcode, Op, ResultTy, DL));
256288
}
257289

258-
VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
259-
Type *ResultTy) {
290+
VPValue *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
291+
Type *ResultTy) {
292+
if (auto *V = Folder.foldCast(Opcode, Op, ResultTy))
293+
if (BB)
294+
return getOrAddLiveIn(V);
260295
return tryInsertInstruction(new VPWidenCastRecipe(Opcode, Op, ResultTy));
261296
}
262297

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
//===- VPlanConstantFolder.h - ConstantFolder for VPlan -------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "VPlanValue.h"
10+
#include "llvm/IR/ConstantFolder.h"
11+
12+
#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANCONSTANTFOLDER_H
13+
#define LLVM_TRANSFORMS_VECTORIZE_VPLANCONSTANTFOLDER_H
14+
15+
namespace llvm {
16+
class VPConstantFolder {
17+
private:
18+
ConstantFolder Folder;
19+
20+
Constant *getIRConstant(VPValue *V) const {
21+
if (!V->isLiveIn())
22+
return nullptr;
23+
return dyn_cast_if_present<Constant>(V->getLiveInIRValue());
24+
}
25+
26+
Value *foldBinOp(Instruction::BinaryOps Opcode, VPValue *LHS,
27+
VPValue *RHS) const {
28+
auto *LC = getIRConstant(LHS);
29+
auto *RC = getIRConstant(RHS);
30+
if (LC && RC)
31+
return Folder.FoldBinOp(Opcode, LC, RC);
32+
return nullptr;
33+
}
34+
35+
public:
36+
Value *foldAnd(VPValue *LHS, VPValue *RHS) const {
37+
return foldBinOp(Instruction::BinaryOps::And, LHS, RHS);
38+
}
39+
40+
Value *foldOr(VPValue *LHS, VPValue *RHS) const {
41+
return foldBinOp(Instruction::BinaryOps::Or, LHS, RHS);
42+
}
43+
44+
Value *foldNot(VPValue *Op) const {
45+
auto *C = getIRConstant(Op);
46+
if (C)
47+
return Folder.FoldBinOp(Instruction::BinaryOps::Xor, C,
48+
Constant::getAllOnesValue(C->getType()));
49+
return nullptr;
50+
}
51+
52+
Value *foldLogicalAnd(VPValue *LHS, VPValue *RHS) const {
53+
auto *LC = getIRConstant(LHS);
54+
auto *RC = getIRConstant(RHS);
55+
if (LC && RC)
56+
return Folder.FoldSelect(LC, RC,
57+
ConstantInt::getNullValue(RC->getType()));
58+
return nullptr;
59+
}
60+
61+
Value *foldSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal) const {
62+
auto *CC = getIRConstant(Cond);
63+
auto *TV = getIRConstant(TrueVal);
64+
auto *FV = getIRConstant(FalseVal);
65+
if (CC && TV && FV)
66+
return Folder.FoldSelect(CC, TV, FV);
67+
return nullptr;
68+
}
69+
70+
Value *foldCmp(CmpInst::Predicate Pred, VPValue *LHS, VPValue *RHS) const {
71+
auto *LC = getIRConstant(LHS);
72+
auto *RC = getIRConstant(RHS);
73+
if (LC && RC)
74+
return Folder.FoldCmp(Pred, LC, RC);
75+
return nullptr;
76+
}
77+
78+
Value *foldPtrAdd(VPValue *Base, VPValue *Offset, GEPNoWrapFlags NW) const {
79+
auto *BC = getIRConstant(Base);
80+
auto *OC = getIRConstant(Offset);
81+
if (BC && OC) {
82+
auto &Ctx = BC->getType()->getContext();
83+
return Folder.FoldGEP(Type::getInt8Ty(Ctx), BC, OC, NW);
84+
}
85+
return nullptr;
86+
}
87+
88+
Value *foldCast(Instruction::CastOps Opcode, VPValue *Op,
89+
Type *DestTy) const {
90+
auto *C = getIRConstant(Op);
91+
if (C)
92+
return Folder.FoldCast(Opcode, C, DestTy);
93+
return nullptr;
94+
}
95+
};
96+
} // namespace llvm
97+
98+
#endif

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -547,8 +547,8 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
547547
VPBuilder &Builder) {
548548
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
549549
VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
550-
VPSingleDefRecipe *BaseIV = Builder.createDerivedIV(
551-
Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
550+
VPValue *BaseIV = Builder.createDerivedIV(Kind, FPBinOp, StartV, CanonicalIV,
551+
Step, "offset.idx");
552552

553553
// Truncate base induction if needed.
554554
Type *CanonicalIVType = CanonicalIV->getScalarType();
@@ -2098,7 +2098,7 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
20982098
auto *CanonicalIVIncrement =
20992099
cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
21002100
Builder.setInsertPoint(CanonicalIVIncrement);
2101-
VPSingleDefRecipe *OpVPEVL = VPEVL;
2101+
VPValue *OpVPEVL = VPEVL;
21022102
if (unsigned IVSize = CanonicalIVPHI->getScalarType()->getScalarSizeInBits();
21032103
IVSize != 32) {
21042104
OpVPEVL = Builder.createScalarCast(

0 commit comments

Comments
 (0)