Skip to content

Commit 0ffa377

Browse files
kazutakahiratakazutakahirata
authored
[ThinLTO] Shrink GlobalValueSummary by 8 bytes (#107342)
During the ThinLTO indexing step for one of our large applications, we create 7.5 million instances of GlobalValueSummary. Changing: std::vector<ValueInfo> RefEdgeList; to: SmallVector<ValueInfo, 0> RefEdgeList; in GlobalValueSummary reduces the size of each instance by 8 bytes. The rest of the patch makes the same change to other places so that the types stay compatible across function boundaries.
1 parent a291fe5 commit 0ffa377

File tree

6 files changed

+45
-40
lines changed

6 files changed

+45
-40
lines changed

‎llvm/include/llvm/AsmParser/LLParser.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -419,7 +419,7 @@ namespace llvm {
419419
bool parseParamAccessCall(FunctionSummary::ParamAccess::Call &Call,
420420
IdLocListType &IdLocList);
421421
bool parseParamAccessOffset(ConstantRange &Range);
422-
bool parseOptionalRefs(std::vector<ValueInfo> &Refs);
422+
bool parseOptionalRefs(SmallVectorImpl<ValueInfo> &Refs);
423423
bool parseTypeIdEntry(unsigned ID);
424424
bool parseTypeIdSummary(TypeIdSummary &TIS);
425425
bool parseTypeIdCompatibleVtableEntry(unsigned ID);

‎llvm/include/llvm/IR/ModuleSummaryIndex.h‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -538,10 +538,13 @@ class GlobalValueSummary {
538538
/// (either by the initializer of a global variable, or referenced
539539
/// from within a function). This does not include functions called, which
540540
/// are listed in the derived FunctionSummary object.
541-
std::vector<ValueInfo> RefEdgeList;
541+
/// We use SmallVector<ValueInfo, 0> instead of std::vector<ValueInfo> for its
542+
/// smaller memory footprint.
543+
SmallVector<ValueInfo, 0> RefEdgeList;
542544

543545
protected:
544-
GlobalValueSummary(SummaryKind K, GVFlags Flags, std::vector<ValueInfo> Refs)
546+
GlobalValueSummary(SummaryKind K, GVFlags Flags,
547+
SmallVectorImpl<ValueInfo> &&Refs)
545548
: Kind(K), Flags(Flags), RefEdgeList(std::move(Refs)) {
546549
assert((K != AliasKind || Refs.empty()) &&
547550
"Expect no references for AliasSummary");
@@ -641,7 +644,7 @@ class AliasSummary : public GlobalValueSummary {
641644

642645
public:
643646
AliasSummary(GVFlags Flags)
644-
: GlobalValueSummary(AliasKind, Flags, ArrayRef<ValueInfo>{}),
647+
: GlobalValueSummary(AliasKind, Flags, SmallVector<ValueInfo, 0>{}),
645648
AliaseeSummary(nullptr) {}
646649

647650
/// Check if this is an alias summary.
@@ -857,7 +860,7 @@ class FunctionSummary : public GlobalValueSummary {
857860
/*NotEligibleToImport=*/true, /*Live=*/true, /*IsLocal=*/false,
858861
/*CanAutoHide=*/false, GlobalValueSummary::ImportKind::Definition),
859862
/*NumInsts=*/0, FunctionSummary::FFlags{}, /*EntryCount=*/0,
860-
std::vector<ValueInfo>(), std::move(Edges),
863+
SmallVector<ValueInfo, 0>(), std::move(Edges),
861864
std::vector<GlobalValue::GUID>(),
862865
std::vector<FunctionSummary::VFuncId>(),
863866
std::vector<FunctionSummary::VFuncId>(),
@@ -913,7 +916,7 @@ class FunctionSummary : public GlobalValueSummary {
913916

914917
public:
915918
FunctionSummary(GVFlags Flags, unsigned NumInsts, FFlags FunFlags,
916-
uint64_t EntryCount, std::vector<ValueInfo> Refs,
919+
uint64_t EntryCount, SmallVectorImpl<ValueInfo> &&Refs,
917920
std::vector<EdgeTy> CGEdges,
918921
std::vector<GlobalValue::GUID> TypeTests,
919922
std::vector<VFuncId> TypeTestAssumeVCalls,
@@ -1166,7 +1169,7 @@ class GlobalVarSummary : public GlobalValueSummary {
11661169
} VarFlags;
11671170

11681171
GlobalVarSummary(GVFlags Flags, GVarFlags VarFlags,
1169-
std::vector<ValueInfo> Refs)
1172+
SmallVectorImpl<ValueInfo> &&Refs)
11701173
: GlobalValueSummary(GlobalVarKind, Flags, std::move(Refs)),
11711174
VarFlags(VarFlags) {}
11721175

‎llvm/include/llvm/IR/ModuleSummaryIndexYAML.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> {
216216
}
217217
auto &Elem = V.try_emplace(KeyInt, /*IsAnalysis=*/false).first->second;
218218
for (auto &FSum : FSums) {
219-
std::vector<ValueInfo> Refs;
219+
SmallVector<ValueInfo, 0> Refs;
220220
Refs.reserve(FSum.Refs.size());
221221
for (auto &RefGUID : FSum.Refs) {
222222
auto It = V.try_emplace(RefGUID, /*IsAnalysis=*/false).first;
@@ -229,9 +229,9 @@ template <> struct CustomMappingTraits<GlobalValueSummaryMapTy> {
229229
FSum.NotEligibleToImport, FSum.Live, FSum.IsLocal,
230230
FSum.CanAutoHide,
231231
static_cast<GlobalValueSummary::ImportKind>(FSum.ImportType)),
232-
/*NumInsts=*/0, FunctionSummary::FFlags{}, /*EntryCount=*/0, Refs,
233-
ArrayRef<FunctionSummary::EdgeTy>{}, std::move(FSum.TypeTests),
234-
std::move(FSum.TypeTestAssumeVCalls),
232+
/*NumInsts=*/0, FunctionSummary::FFlags{}, /*EntryCount=*/0,
233+
std::move(Refs), ArrayRef<FunctionSummary::EdgeTy>{},
234+
std::move(FSum.TypeTests), std::move(FSum.TypeTestAssumeVCalls),
235235
std::move(FSum.TypeCheckedLoadVCalls),
236236
std::move(FSum.TypeTestAssumeConstVCalls),
237237
std::move(FSum.TypeCheckedLoadConstVCalls),

‎llvm/lib/Analysis/ModuleSummaryAnalysis.cpp‎

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -99,10 +99,11 @@ extern cl::opt<bool> MemProfReportHintedSizes;
9999
// can only take an address of basic block located in the same function.
100100
// Set `RefLocalLinkageIFunc` to true if the analyzed value references a
101101
// local-linkage ifunc.
102-
static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
103-
SetVector<ValueInfo, std::vector<ValueInfo>> &RefEdges,
104-
SmallPtrSet<const User *, 8> &Visited,
105-
bool &RefLocalLinkageIFunc) {
102+
static bool
103+
findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
104+
SetVector<ValueInfo, SmallVector<ValueInfo, 0>> &RefEdges,
105+
SmallPtrSet<const User *, 8> &Visited,
106+
bool &RefLocalLinkageIFunc) {
106107
bool HasBlockAddress = false;
107108
SmallVector<const User *, 32> Worklist;
108109
if (Visited.insert(CurUser).second)
@@ -310,7 +311,7 @@ static void computeFunctionSummary(
310311
MapVector<ValueInfo, CalleeInfo, DenseMap<ValueInfo, unsigned>,
311312
std::vector<std::pair<ValueInfo, CalleeInfo>>>
312313
CallGraphEdges;
313-
SetVector<ValueInfo, std::vector<ValueInfo>> RefEdges, LoadRefEdges,
314+
SetVector<ValueInfo, SmallVector<ValueInfo, 0>> RefEdges, LoadRefEdges,
314315
StoreRefEdges;
315316
SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> TypeTests;
316317
SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
@@ -568,16 +569,17 @@ static void computeFunctionSummary(
568569
if (PSI->hasPartialSampleProfile() && ScalePartialSampleProfileWorkingSetSize)
569570
Index.addBlockCount(F.size());
570571

571-
std::vector<ValueInfo> Refs;
572+
SmallVector<ValueInfo, 0> Refs;
572573
if (IsThinLTO) {
573-
auto AddRefEdges = [&](const std::vector<const Instruction *> &Instrs,
574-
SetVector<ValueInfo, std::vector<ValueInfo>> &Edges,
575-
SmallPtrSet<const User *, 8> &Cache) {
576-
for (const auto *I : Instrs) {
577-
Cache.erase(I);
578-
findRefEdges(Index, I, Edges, Cache, HasLocalIFuncCallOrRef);
579-
}
580-
};
574+
auto AddRefEdges =
575+
[&](const std::vector<const Instruction *> &Instrs,
576+
SetVector<ValueInfo, SmallVector<ValueInfo, 0>> &Edges,
577+
SmallPtrSet<const User *, 8> &Cache) {
578+
for (const auto *I : Instrs) {
579+
Cache.erase(I);
580+
findRefEdges(Index, I, Edges, Cache, HasLocalIFuncCallOrRef);
581+
}
582+
};
581583

582584
// By now we processed all instructions in a function, except
583585
// non-volatile loads and non-volatile value stores. Let's find
@@ -805,7 +807,7 @@ static void computeVariableSummary(ModuleSummaryIndex &Index,
805807
DenseSet<GlobalValue::GUID> &CantBePromoted,
806808
const Module &M,
807809
SmallVectorImpl<MDNode *> &Types) {
808-
SetVector<ValueInfo, std::vector<ValueInfo>> RefEdges;
810+
SetVector<ValueInfo, SmallVector<ValueInfo, 0>> RefEdges;
809811
SmallPtrSet<const User *, 8> Visited;
810812
bool RefLocalIFunc = false;
811813
bool HasBlockAddress =
@@ -961,7 +963,7 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
961963
/* MayThrow */ true,
962964
/* HasUnknownCall */ true,
963965
/* MustBeUnreachable */ false},
964-
/*EntryCount=*/0, ArrayRef<ValueInfo>{},
966+
/*EntryCount=*/0, SmallVector<ValueInfo, 0>{},
965967
ArrayRef<FunctionSummary::EdgeTy>{},
966968
ArrayRef<GlobalValue::GUID>{},
967969
ArrayRef<FunctionSummary::VFuncId>{},
@@ -978,7 +980,7 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
978980
GlobalVarSummary::GVarFlags(
979981
false, false, cast<GlobalVariable>(GV)->isConstant(),
980982
GlobalObject::VCallVisibilityPublic),
981-
ArrayRef<ValueInfo>{});
983+
SmallVector<ValueInfo, 0>{});
982984
Index.addGlobalValueSummary(*GV, std::move(Summary));
983985
}
984986
});

‎llvm/lib/AsmParser/LLParser.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9399,7 +9399,7 @@ bool LLParser::parseFunctionSummary(std::string Name, GlobalValue::GUID GUID,
93999399
std::vector<FunctionSummary::EdgeTy> Calls;
94009400
FunctionSummary::TypeIdInfo TypeIdInfo;
94019401
std::vector<FunctionSummary::ParamAccess> ParamAccesses;
9402-
std::vector<ValueInfo> Refs;
9402+
SmallVector<ValueInfo, 0> Refs;
94039403
std::vector<CallsiteInfo> Callsites;
94049404
std::vector<AllocInfo> Allocs;
94059405
// Default is all-zeros (conservative values).
@@ -9487,7 +9487,7 @@ bool LLParser::parseVariableSummary(std::string Name, GlobalValue::GUID GUID,
94879487
/* WriteOnly */ false,
94889488
/* Constant */ false,
94899489
GlobalObject::VCallVisibilityPublic);
9490-
std::vector<ValueInfo> Refs;
9490+
SmallVector<ValueInfo, 0> Refs;
94919491
VTableFuncList VTableFuncs;
94929492
if (parseToken(lltok::colon, "expected ':' here") ||
94939493
parseToken(lltok::lparen, "expected '(' here") ||
@@ -9993,7 +9993,7 @@ bool LLParser::parseOptionalParamAccesses(
99939993

99949994
/// OptionalRefs
99959995
/// := 'refs' ':' '(' GVReference [',' GVReference]* ')'
9996-
bool LLParser::parseOptionalRefs(std::vector<ValueInfo> &Refs) {
9996+
bool LLParser::parseOptionalRefs(SmallVectorImpl<ValueInfo> &Refs) {
99979997
assert(Lex.getKind() == lltok::kw_refs);
99989998
Lex.Lex();
99999999

‎llvm/lib/Bitcode/Reader/BitcodeReader.cpp‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -985,7 +985,7 @@ class ModuleSummaryIndexBitcodeReader : public BitcodeReaderBase {
985985
Error parseValueSymbolTable(
986986
uint64_t Offset,
987987
DenseMap<unsigned, GlobalValue::LinkageTypes> &ValueIdToLinkageMap);
988-
std::vector<ValueInfo> makeRefList(ArrayRef<uint64_t> Record);
988+
SmallVector<ValueInfo, 0> makeRefList(ArrayRef<uint64_t> Record);
989989
std::vector<FunctionSummary::EdgeTy> makeCallList(ArrayRef<uint64_t> Record,
990990
bool IsOldProfileFormat,
991991
bool HasProfile,
@@ -7369,9 +7369,9 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() {
73697369
}
73707370
}
73717371

7372-
std::vector<ValueInfo>
7372+
SmallVector<ValueInfo, 0>
73737373
ModuleSummaryIndexBitcodeReader::makeRefList(ArrayRef<uint64_t> Record) {
7374-
std::vector<ValueInfo> Ret;
7374+
SmallVector<ValueInfo, 0> Ret;
73757375
Ret.reserve(Record.size());
73767376
for (uint64_t RefValueId : Record)
73777377
Ret.push_back(std::get<0>(getValueInfoFromValueId(RefValueId)));
@@ -7516,7 +7516,7 @@ void ModuleSummaryIndexBitcodeReader::parseTypeIdCompatibleVtableSummaryRecord(
75167516
parseTypeIdCompatibleVtableInfo(Record, Slot, TypeId);
75177517
}
75187518

7519-
static void setSpecialRefs(std::vector<ValueInfo> &Refs, unsigned ROCnt,
7519+
static void setSpecialRefs(SmallVectorImpl<ValueInfo> &Refs, unsigned ROCnt,
75207520
unsigned WOCnt) {
75217521
// Readonly and writeonly refs are in the end of the refs list.
75227522
assert(ROCnt + WOCnt <= Refs.size());
@@ -7666,7 +7666,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
76667666
int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs;
76677667
assert(Record.size() >= RefListStartIndex + NumRefs &&
76687668
"Record size inconsistent with number of references");
7669-
std::vector<ValueInfo> Refs = makeRefList(
7669+
SmallVector<ValueInfo, 0> Refs = makeRefList(
76707670
ArrayRef<uint64_t>(Record).slice(RefListStartIndex, NumRefs));
76717671
bool HasProfile = (BitCode == bitc::FS_PERMODULE_PROFILE);
76727672
bool HasRelBF = (BitCode == bitc::FS_PERMODULE_RELBF);
@@ -7741,7 +7741,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
77417741
GVF = getDecodedGVarFlags(Record[2]);
77427742
RefArrayStart = 3;
77437743
}
7744-
std::vector<ValueInfo> Refs =
7744+
SmallVector<ValueInfo, 0> Refs =
77457745
makeRefList(ArrayRef<uint64_t>(Record).slice(RefArrayStart));
77467746
auto FS =
77477747
std::make_unique<GlobalVarSummary>(Flags, GVF, std::move(Refs));
@@ -7762,7 +7762,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
77627762
unsigned RefListStartIndex = 4;
77637763
unsigned VTableListStartIndex = RefListStartIndex + NumRefs;
77647764
auto Flags = getDecodedGVSummaryFlags(RawFlags, Version);
7765-
std::vector<ValueInfo> Refs = makeRefList(
7765+
SmallVector<ValueInfo, 0> Refs = makeRefList(
77667766
ArrayRef<uint64_t>(Record).slice(RefListStartIndex, NumRefs));
77677767
VTableFuncList VTableFuncs;
77687768
for (unsigned I = VTableListStartIndex, E = Record.size(); I != E; ++I) {
@@ -7823,7 +7823,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
78237823
int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs;
78247824
assert(Record.size() >= RefListStartIndex + NumRefs &&
78257825
"Record size inconsistent with number of references");
7826-
std::vector<ValueInfo> Refs = makeRefList(
7826+
SmallVector<ValueInfo, 0> Refs = makeRefList(
78277827
ArrayRef<uint64_t>(Record).slice(RefListStartIndex, NumRefs));
78287828
bool HasProfile = (BitCode == bitc::FS_COMBINED_PROFILE);
78297829
std::vector<FunctionSummary::EdgeTy> Edges = makeCallList(
@@ -7883,7 +7883,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
78837883
GVF = getDecodedGVarFlags(Record[3]);
78847884
RefArrayStart = 4;
78857885
}
7886-
std::vector<ValueInfo> Refs =
7886+
SmallVector<ValueInfo, 0> Refs =
78877887
makeRefList(ArrayRef<uint64_t>(Record).slice(RefArrayStart));
78887888
auto FS =
78897889
std::make_unique<GlobalVarSummary>(Flags, GVF, std::move(Refs));

0 commit comments

Comments
 (0)