Skip to content

Commit 49b408e

Browse files
StevenYangCCStevenYangCC
committed
[SinkGEPConstOffset] FEAT: Sink constant offsets down a GEP chain to tail for reduction of register usage.
Summary: Sink constant offsets down the GEP chain to the tail helps reduce register usage. For example: %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 512 %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %ofst0 %gep2 = getelementptr half, ptr addrspace(3) %gep1, i32 %ofst1 %data = load half, ptr addrspace(3) %gep2, align 2 ==> %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 %ofst0 %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %ofst1 %gep2 = getelementptr half, ptr addrspace(3) %gep1, i32 512 %data = load half, ptr addrspace(3) %gep2, align 2
1 parent 2df5027 commit 49b408e

File tree

18 files changed

+846
-529
lines changed

18 files changed

+846
-529
lines changed

‎llvm/include/llvm/InitializePasses.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,7 @@ LLVM_ABI void initializeScavengerTestPass(PassRegistry &);
298298
LLVM_ABI void initializeScopedNoAliasAAWrapperPassPass(PassRegistry &);
299299
LLVM_ABI void
300300
initializeSeparateConstOffsetFromGEPLegacyPassPass(PassRegistry &);
301+
LLVM_ABI void initializeSinkGEPConstOffsetLegacyPassPass(PassRegistry &);
301302
LLVM_ABI void initializeShadowStackGCLoweringPass(PassRegistry &);
302303
LLVM_ABI void initializeShrinkWrapLegacyPass(PassRegistry &);
303304
LLVM_ABI void initializeSingleLoopExtractorPass(PassRegistry &);

‎llvm/include/llvm/LinkAllPasses.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ struct ForcePassLinking {
134134
(void)llvm::createPartiallyInlineLibCallsPass();
135135
(void)llvm::createScalarizerPass();
136136
(void)llvm::createSeparateConstOffsetFromGEPPass();
137+
(void)llvm::createSinkGEPConstOffsetPass();
137138
(void)llvm::createSpeculativeExecutionPass();
138139
(void)llvm::createSpeculativeExecutionIfHasBranchDivergencePass();
139140
(void)llvm::createStraightLineStrengthReducePass();

‎llvm/include/llvm/Transforms/Scalar.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,13 @@ LLVM_ABI FunctionPass *createPartiallyInlineLibCallsPass();
168168
LLVM_ABI FunctionPass *
169169
createSeparateConstOffsetFromGEPPass(bool LowerGEP = false);
170170

171+
//===----------------------------------------------------------------------===//
172+
//
173+
// SinkGEPConstOffset - Sink constant offsets down the GEP chain to the tail for
174+
// reduction of register usage.
175+
//
176+
FunctionPass *createSinkGEPConstOffsetPass();
177+
171178
//===----------------------------------------------------------------------===//
172179
//
173180
// SpeculativeExecution - Aggressively hoist instructions to enable

‎llvm/include/llvm/Transforms/Scalar/SinkGEPConstOffset.h‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
//===- SinkGEPConstOffset.h -----------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_TRANSFORMS_SCALAR_SINKGEPCONSTOFFSET_H
10+
#define LLVM_TRANSFORMS_SCALAR_SINKGEPCONSTOFFSET_H
11+
12+
#include "llvm/IR/PassManager.h"
13+
14+
namespace llvm {
15+
16+
class SinkGEPConstOffsetPass
17+
: public PassInfoMixin<SinkGEPConstOffsetPass> {
18+
public:
19+
SinkGEPConstOffsetPass() {}
20+
void printPipeline(raw_ostream &OS,
21+
function_ref<StringRef(StringRef)> MapClassName2PassName);
22+
PreservedAnalyses run(Function &F, FunctionAnalysisManager &);
23+
};
24+
25+
} // end namespace llvm
26+
27+
#endif // LLVM_TRANSFORMS_SCALAR_SINKGEPCONSTOFFSET_H

‎llvm/lib/Passes/PassBuilder.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,7 @@
323323
#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
324324
#include "llvm/Transforms/Scalar/Scalarizer.h"
325325
#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
326+
#include "llvm/Transforms/Scalar/SinkGEPConstOffset.h"
326327
#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
327328
#include "llvm/Transforms/Scalar/SimplifyCFG.h"
328329
#include "llvm/Transforms/Scalar/Sink.h"

‎llvm/lib/Passes/PassRegistry.def‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -476,6 +476,8 @@ FUNCTION_PASS("sccp", SCCPPass())
476476
FUNCTION_PASS("select-optimize", SelectOptimizePass(TM))
477477
FUNCTION_PASS("separate-const-offset-from-gep",
478478
SeparateConstOffsetFromGEPPass())
479+
FUNCTION_PASS("sink-gep-const-offset",
480+
SinkGEPConstOffsetPass())
479481
FUNCTION_PASS("sink", SinkingPass())
480482
FUNCTION_PASS("sjlj-eh-prepare", SjLjEHPreparePass(TM))
481483
FUNCTION_PASS("slp-vectorizer", SLPVectorizerPass())

‎llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
105105
#include "llvm/Transforms/Scalar/NaryReassociate.h"
106106
#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
107+
#include "llvm/Transforms/Scalar/SinkGEPConstOffset.h"
107108
#include "llvm/Transforms/Scalar/Sink.h"
108109
#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
109110
#include "llvm/Transforms/Scalar/StructurizeCFG.h"
@@ -1238,6 +1239,7 @@ void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {
12381239
if (isPassEnabled(EnableLoopPrefetch, CodeGenOptLevel::Aggressive))
12391240
addPass(createLoopDataPrefetchPass());
12401241
addPass(createSeparateConstOffsetFromGEPPass());
1242+
addPass(createSinkGEPConstOffsetPass());
12411243
// ReassociateGEPs exposes more opportunities for SLSR. See
12421244
// the example in reassociate-geps-and-slsr.ll.
12431245
addPass(createStraightLineStrengthReducePass());
@@ -2310,6 +2312,8 @@ void AMDGPUCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
23102312

23112313
addPass(SeparateConstOffsetFromGEPPass());
23122314

2315+
addPass(SinkGEPConstOffsetPass());
2316+
23132317
// ReassociateGEPs exposes more opportunities for SLSR. See
23142318
// the example in reassociate-geps-and-slsr.ll.
23152319
addPass(StraightLineStrengthReducePass());

‎llvm/lib/Transforms/Scalar/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ add_llvm_component_library(LLVMScalarOpts
7171
Scalarizer.cpp
7272
ScalarizeMaskedMemIntrin.cpp
7373
SeparateConstOffsetFromGEP.cpp
74+
SinkGEPConstOffset.cpp
7475
SimpleLoopUnswitch.cpp
7576
SimplifyCFGPass.cpp
7677
Sink.cpp

‎llvm/lib/Transforms/Scalar/Scalar.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
4545
initializeSinkingLegacyPassPass(Registry);
4646
initializeTailCallElimPass(Registry);
4747
initializeSeparateConstOffsetFromGEPLegacyPassPass(Registry);
48+
initializeSinkGEPConstOffsetLegacyPassPass(Registry);
4849
initializeSpeculativeExecutionLegacyPassPass(Registry);
4950
initializeStraightLineStrengthReduceLegacyPassPass(Registry);
5051
initializePlaceBackedgeSafepointsLegacyPassPass(Registry);

‎llvm/lib/Transforms/Scalar/SinkGEPConstOffset.cpp‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
//===- SinkGEPConstOffset.cpp -------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include "llvm/Transforms/Scalar/SinkGEPConstOffset.h"
10+
#include "llvm/ADT/APInt.h"
11+
#include "llvm/ADT/DenseMap.h"
12+
#include "llvm/ADT/DepthFirstIterator.h"
13+
#include "llvm/ADT/SmallVector.h"
14+
#include "llvm/Analysis/LoopInfo.h"
15+
#include "llvm/Analysis/MemoryBuiltins.h"
16+
#include "llvm/Analysis/TargetLibraryInfo.h"
17+
#include "llvm/Analysis/TargetTransformInfo.h"
18+
#include "llvm/Analysis/ValueTracking.h"
19+
#include "llvm/IR/BasicBlock.h"
20+
#include "llvm/IR/Constant.h"
21+
#include "llvm/IR/Constants.h"
22+
#include "llvm/IR/DataLayout.h"
23+
#include "llvm/IR/DerivedTypes.h"
24+
#include "llvm/IR/Dominators.h"
25+
#include "llvm/IR/Function.h"
26+
#include "llvm/IR/GetElementPtrTypeIterator.h"
27+
#include "llvm/IR/IRBuilder.h"
28+
#include "llvm/IR/InstrTypes.h"
29+
#include "llvm/IR/Instruction.h"
30+
#include "llvm/IR/Instructions.h"
31+
#include "llvm/IR/Module.h"
32+
#include "llvm/IR/PassManager.h"
33+
#include "llvm/IR/PatternMatch.h"
34+
#include "llvm/IR/Type.h"
35+
#include "llvm/IR/User.h"
36+
#include "llvm/IR/Value.h"
37+
#include "llvm/InitializePasses.h"
38+
#include "llvm/Pass.h"
39+
#include "llvm/Support/Casting.h"
40+
#include "llvm/Support/CommandLine.h"
41+
#include "llvm/Support/ErrorHandling.h"
42+
#include "llvm/Support/raw_ostream.h"
43+
#include "llvm/Transforms/Scalar.h"
44+
#include "llvm/Transforms/Utils/Local.h"
45+
#include <cassert>
46+
#include <cstdint>
47+
#include <string>
48+
49+
using namespace llvm;
50+
using namespace llvm::PatternMatch;
51+
52+
namespace {
53+
54+
/// A pass that tries to sink const offset in GEP chain to tail.
55+
/// It is a FunctionPass because searching for the constant offset may inspect
56+
/// other basic blocks.
57+
class SinkGEPConstOffsetLegacyPass : public FunctionPass {
58+
public:
59+
static char ID;
60+
61+
SinkGEPConstOffsetLegacyPass() : FunctionPass(ID) {
62+
initializeSinkGEPConstOffsetLegacyPassPass(
63+
*PassRegistry::getPassRegistry());
64+
}
65+
66+
void getAnalysisUsage(AnalysisUsage &AU) const override {
67+
AU.setPreservesCFG();
68+
}
69+
70+
bool runOnFunction(Function &F) override;
71+
};
72+
73+
} // end anonymous namespace
74+
75+
char SinkGEPConstOffsetLegacyPass::ID = 0;
76+
77+
INITIALIZE_PASS_BEGIN(
78+
SinkGEPConstOffsetLegacyPass, "sink-gep-const-offset",
79+
"Sink const offsets down the GEP chain to the tail for reduction of "
80+
"register usage", false, false)
81+
INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
82+
INITIALIZE_PASS_END(
83+
SinkGEPConstOffsetLegacyPass, "sink-gep-const-offset",
84+
"Sink const offsets down the GEP chain to the tail for reduction of "
85+
"register usage", false, false)
86+
87+
FunctionPass *llvm::createSinkGEPConstOffsetPass() {
88+
return new SinkGEPConstOffsetLegacyPass();
89+
}
90+
91+
/// The purpose of this function is to sink the constant offsets in the base
92+
/// GEP to current GEP.
93+
///
94+
/// A simple example is given:
95+
///
96+
/// %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 512
97+
/// %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %ofst0
98+
/// %data = load half, ptr addrspace(3) %gep1, align 2
99+
/// ==>
100+
/// %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 %ofst0
101+
/// %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 512
102+
/// %data = load half, ptr addrspace(3) %gep1, align 2
103+
static bool sinkGEPConstantOffset(Value *Ptr, const DataLayout *DL) {
104+
GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
105+
if (!GEP)
106+
return false;
107+
108+
if (GEP->getNumIndices() != 1)
109+
return false;
110+
111+
GetElementPtrInst *BaseGEP =
112+
dyn_cast<GetElementPtrInst>(GEP->getPointerOperand());
113+
if (!BaseGEP)
114+
return false;
115+
116+
if (BaseGEP->getNumIndices() != 1)
117+
return false;
118+
119+
Value *Idx = GEP->getOperand(1);
120+
Value *BaseIdx = BaseGEP->getOperand(1);
121+
122+
ConstantInt *BaseIdxAsCI = dyn_cast<ConstantInt>(BaseIdx);
123+
if (!BaseIdxAsCI)
124+
return false;
125+
126+
Type *ResTy = GEP->getResultElementType();
127+
Type *BaseResTy = BaseGEP->getResultElementType();
128+
129+
ConstantInt *IdxAsCI = dyn_cast<ConstantInt>(Idx);
130+
if (IdxAsCI) {
131+
// %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 8
132+
// %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 4
133+
// as:
134+
// %gep1 = getelementptr half, ptr addrspace(3) %ptr, i32 12
135+
Type *NewResTy = nullptr;
136+
int64_t NewIdxValue = 0;
137+
if (ResTy == BaseResTy) {
138+
NewResTy = ResTy;
139+
NewIdxValue = BaseIdxAsCI->getSExtValue() + IdxAsCI->getSExtValue();
140+
} else {
141+
NewResTy = Type::getInt8Ty(GEP->getContext());
142+
NewIdxValue = (BaseIdxAsCI->getSExtValue() *
143+
DL->getTypeAllocSize(BaseResTy)) +
144+
(IdxAsCI->getSExtValue() *
145+
DL->getTypeAllocSize(ResTy));
146+
}
147+
assert(NewResTy);
148+
Type *NewIdxType = (Idx->getType()->getPrimitiveSizeInBits() >
149+
BaseIdx->getType()->getPrimitiveSizeInBits())
150+
? Idx->getType() : BaseIdx->getType();
151+
Constant *NewIdx = ConstantInt::get(NewIdxType, NewIdxValue);
152+
auto *NewGEP = GetElementPtrInst::Create(
153+
NewResTy, BaseGEP->getPointerOperand(), NewIdx);
154+
NewGEP->setIsInBounds(GEP->isInBounds());
155+
NewGEP->insertBefore(GEP->getIterator());
156+
NewGEP->takeName(GEP);
157+
158+
GEP->replaceAllUsesWith(NewGEP);
159+
RecursivelyDeleteTriviallyDeadInstructions(GEP);
160+
161+
return true;
162+
}
163+
164+
// %gep0 = getelementptr half, ptr addrspace(3) %ptr, i32 8
165+
// %gep1 = getelementptr half, ptr addrspace(3) %gep0, i32 %idx
166+
// as:
167+
// %gepx0 = getelementptr half, ptr addrspace(3) %ptr, i32 %idx
168+
// %gepx1 = getelementptr half, ptr addrspace(3) %gepx0, i32 8
169+
auto *GEPX0 =
170+
GetElementPtrInst::Create(ResTy, BaseGEP->getPointerOperand(), Idx);
171+
GEPX0->insertBefore(GEP->getIterator());
172+
auto *GEPX1 = GetElementPtrInst::Create(BaseResTy, GEPX0, BaseIdx);
173+
GEPX1->setIsInBounds(GEP->isInBounds());
174+
GEPX1->insertBefore(GEP->getIterator());
175+
GEPX1->takeName(GEP);
176+
177+
GEP->replaceAllUsesWith(GEPX1);
178+
RecursivelyDeleteTriviallyDeadInstructions(GEP);
179+
180+
return true;
181+
}
182+
183+
static bool sinkGEPConstantOffset(Function &F) {
184+
const DataLayout *DL = &F.getDataLayout();
185+
ReversePostOrderTraversal<Function*> RPOT(&F);
186+
bool Changed = false;
187+
for (auto *BB : RPOT)
188+
for (Instruction &I : llvm::make_early_inc_range(*BB))
189+
if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I))
190+
Changed |= sinkGEPConstantOffset(GEP, DL);
191+
192+
return Changed;
193+
}
194+
195+
bool SinkGEPConstOffsetLegacyPass::runOnFunction(Function &F) {
196+
if (skipFunction(F))
197+
return false;
198+
199+
return sinkGEPConstantOffset(F);
200+
}
201+
202+
void SinkGEPConstOffsetPass::printPipeline(
203+
raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
204+
static_cast<PassInfoMixin<SinkGEPConstOffsetPass> *>(this)
205+
->printPipeline(OS, MapClassName2PassName);
206+
}
207+
208+
PreservedAnalyses
209+
SinkGEPConstOffsetPass::run(Function &F, FunctionAnalysisManager &AM) {
210+
if (!sinkGEPConstantOffset(F))
211+
return PreservedAnalyses::all();
212+
213+
PreservedAnalyses PA;
214+
PA.preserveSet<CFGAnalyses>();
215+
return PA;
216+
}

0 commit comments

Comments
 (0)