Skip to content

Commit e9368a0

Browse files
rlavaeerlavaee
authored
[SHT_LLVM_BB_ADDR] Implement ELF and YAML support for Propeller CFG data in PGO analysis map. (#164914)
This PR implements the ELF support for PostLink CFG in PGO analysis map as discussed in [RFC](https://discourse.llvm.org/t/rfc-extending-the-pgo-analysis-map-with-propeller-cfg-frequencies/88617/2). A later PR will implement the Codegen Support.
1 parent fe52f1d commit e9368a0

File tree

15 files changed

+242
-128
lines changed

15 files changed

+242
-128
lines changed

‎llvm/include/llvm/Object/ELFTypes.h‎

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -834,30 +834,32 @@ struct BBAddrMap {
834834
bool OmitBBEntries : 1;
835835
bool CallsiteEndOffsets : 1;
836836
bool BBHash : 1;
837+
bool PostLinkCfg : 1;
837838

838839
bool hasPGOAnalysis() const { return FuncEntryCount || BBFreq || BrProb; }
839840

840841
bool hasPGOAnalysisBBData() const { return BBFreq || BrProb; }
841842

842843
// Encodes to minimum bit width representation.
843-
uint8_t encode() const {
844-
return (static_cast<uint8_t>(FuncEntryCount) << 0) |
845-
(static_cast<uint8_t>(BBFreq) << 1) |
846-
(static_cast<uint8_t>(BrProb) << 2) |
847-
(static_cast<uint8_t>(MultiBBRange) << 3) |
848-
(static_cast<uint8_t>(OmitBBEntries) << 4) |
849-
(static_cast<uint8_t>(CallsiteEndOffsets) << 5) |
850-
(static_cast<uint8_t>(BBHash) << 6);
844+
uint16_t encode() const {
845+
return (static_cast<uint16_t>(FuncEntryCount) << 0) |
846+
(static_cast<uint16_t>(BBFreq) << 1) |
847+
(static_cast<uint16_t>(BrProb) << 2) |
848+
(static_cast<uint16_t>(MultiBBRange) << 3) |
849+
(static_cast<uint16_t>(OmitBBEntries) << 4) |
850+
(static_cast<uint16_t>(CallsiteEndOffsets) << 5) |
851+
(static_cast<uint16_t>(BBHash) << 6) |
852+
(static_cast<uint16_t>(PostLinkCfg) << 7);
851853
}
852854

853855
// Decodes from minimum bit width representation and validates no
854856
// unnecessary bits are used.
855-
static Expected<Features> decode(uint8_t Val) {
857+
static Expected<Features> decode(uint16_t Val) {
856858
Features Feat{
857859
static_cast<bool>(Val & (1 << 0)), static_cast<bool>(Val & (1 << 1)),
858860
static_cast<bool>(Val & (1 << 2)), static_cast<bool>(Val & (1 << 3)),
859861
static_cast<bool>(Val & (1 << 4)), static_cast<bool>(Val & (1 << 5)),
860-
static_cast<bool>(Val & (1 << 6))};
862+
static_cast<bool>(Val & (1 << 6)), static_cast<bool>(Val & (1 << 7))};
861863
if (Feat.encode() != Val)
862864
return createStringError(
863865
std::error_code(), "invalid encoding for BBAddrMap::Features: 0x%x",
@@ -867,10 +869,11 @@ struct BBAddrMap {
867869

868870
bool operator==(const Features &Other) const {
869871
return std::tie(FuncEntryCount, BBFreq, BrProb, MultiBBRange,
870-
OmitBBEntries, CallsiteEndOffsets, BBHash) ==
872+
OmitBBEntries, CallsiteEndOffsets, BBHash, PostLinkCfg) ==
871873
std::tie(Other.FuncEntryCount, Other.BBFreq, Other.BrProb,
872874
Other.MultiBBRange, Other.OmitBBEntries,
873-
Other.CallsiteEndOffsets, Other.BBHash);
875+
Other.CallsiteEndOffsets, Other.BBHash,
876+
Other.PostLinkCfg);
874877
}
875878
};
876879

@@ -1010,23 +1013,30 @@ struct PGOAnalysisMap {
10101013
/// probability associated with it.
10111014
struct SuccessorEntry {
10121015
/// Unique ID of this successor basic block.
1013-
uint32_t ID;
1016+
uint32_t ID = 0;
10141017
/// Branch Probability of the edge to this successor taken from MBPI.
10151018
BranchProbability Prob;
1019+
/// Raw edge count from the post link profile (e.g., from bolt or
1020+
/// propeller).
1021+
uint64_t PostLinkFreq = 0;
10161022

10171023
bool operator==(const SuccessorEntry &Other) const {
1018-
return std::tie(ID, Prob) == std::tie(Other.ID, Other.Prob);
1024+
return std::tie(ID, Prob, PostLinkFreq) ==
1025+
std::tie(Other.ID, Other.Prob, Other.PostLinkFreq);
10191026
}
10201027
};
10211028

10221029
/// Block frequency taken from MBFI
10231030
BlockFrequency BlockFreq;
1031+
/// Raw block count taken from the post link profile (e.g., from bolt or
1032+
/// propeller).
1033+
uint64_t PostLinkBlockFreq = 0;
10241034
/// List of successors of the current block
10251035
llvm::SmallVector<SuccessorEntry, 2> Successors;
10261036

10271037
bool operator==(const PGOBBEntry &Other) const {
1028-
return std::tie(BlockFreq, Successors) ==
1029-
std::tie(Other.BlockFreq, Other.Successors);
1038+
return std::tie(BlockFreq, PostLinkBlockFreq, Successors) ==
1039+
std::tie(Other.BlockFreq, PostLinkBlockFreq, Other.Successors);
10301040
}
10311041
};
10321042

‎llvm/include/llvm/ObjectYAML/ELFYAML.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ struct BBAddrMapEntry {
166166
std::optional<llvm::yaml::Hex64> Hash;
167167
};
168168
uint8_t Version;
169-
llvm::yaml::Hex8 Feature;
169+
llvm::yaml::Hex16 Feature;
170170

171171
struct BBRangeEntry {
172172
llvm::yaml::Hex64 BaseAddress;
@@ -203,8 +203,10 @@ struct PGOAnalysisMapEntry {
203203
struct SuccessorEntry {
204204
uint32_t ID;
205205
llvm::yaml::Hex32 BrProb;
206+
std::optional<uint32_t> PostLinkBrFreq;
206207
};
207208
std::optional<uint64_t> BBFreq;
209+
std::optional<uint32_t> PostLinkBBFreq;
208210
std::optional<std::vector<SuccessorEntry>> Successors;
209211
};
210212
std::optional<uint64_t> FuncEntryCount;

‎llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1443,7 +1443,7 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges,
14431443
MF.hasBBSections() && NumMBBSectionRanges > 1,
14441444
// Use static_cast to avoid breakage of tests on windows.
14451445
static_cast<bool>(BBAddrMapSkipEmitBBEntries), HasCalls,
1446-
static_cast<bool>(EmitBBHash)};
1446+
static_cast<bool>(EmitBBHash), false};
14471447
}
14481448

14491449
void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {

‎llvm/lib/Object/ELF.cpp‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -831,17 +831,17 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
831831
};
832832

833833
uint8_t Version = 0;
834-
uint8_t Feature = 0;
834+
uint16_t Feature = 0;
835835
BBAddrMap::Features FeatEnable{};
836836
while (!ULEBSizeErr && !MetadataDecodeErr && Cur &&
837837
Cur.tell() < Content.size()) {
838838
Version = Data.getU8(Cur);
839839
if (!Cur)
840840
break;
841-
if (Version < 2 || Version > 4)
841+
if (Version < 2 || Version > 5)
842842
return createError("unsupported SHT_LLVM_BB_ADDR_MAP version: " +
843843
Twine(static_cast<int>(Version)));
844-
Feature = Data.getU8(Cur); // Feature byte
844+
Feature = Version < 5 ? Data.getU8(Cur) : Data.getU16(Cur);
845845
if (!Cur)
846846
break;
847847
auto FeatEnableOrErr = BBAddrMap::Features::decode(Feature);
@@ -858,6 +858,11 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
858858
"basic block hash feature is enabled: version = " +
859859
Twine(static_cast<int>(Version)) +
860860
" feature = " + Twine(static_cast<int>(Feature)));
861+
if (FeatEnable.PostLinkCfg && Version < 5)
862+
return createError("version should be >= 5 for SHT_LLVM_BB_ADDR_MAP when "
863+
"post link cfg feature is enabled: version = " +
864+
Twine(static_cast<int>(Version)) +
865+
" feature = " + Twine(static_cast<int>(Feature)));
861866
uint32_t NumBlocksInBBRange = 0;
862867
uint32_t NumBBRanges = 1;
863868
typename ELFFile<ELFT>::uintX_t RangeBaseAddress = 0;
@@ -946,6 +951,10 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
946951
uint64_t BBF = FeatEnable.BBFreq
947952
? readULEB128As<uint64_t>(Data, Cur, ULEBSizeErr)
948953
: 0;
954+
uint32_t PostLinkBBFreq =
955+
FeatEnable.PostLinkCfg
956+
? readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr)
957+
: 0;
949958

950959
// Branch probability
951960
llvm::SmallVector<PGOAnalysisMap::PGOBBEntry::SuccessorEntry, 2>
@@ -955,13 +964,20 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
955964
for (uint64_t I = 0; I < SuccCount; ++I) {
956965
uint32_t BBID = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
957966
uint32_t BrProb = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
967+
uint32_t PostLinkFreq =
968+
FeatEnable.PostLinkCfg
969+
? readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr)
970+
: 0;
971+
958972
if (PGOAnalyses)
959-
Successors.push_back({BBID, BranchProbability::getRaw(BrProb)});
973+
Successors.push_back(
974+
{BBID, BranchProbability::getRaw(BrProb), PostLinkFreq});
960975
}
961976
}
962977

963978
if (PGOAnalyses)
964-
PGOBBEntries.push_back({BlockFrequency(BBF), std::move(Successors)});
979+
PGOBBEntries.push_back(
980+
{BlockFrequency(BBF), PostLinkBBFreq, std::move(Successors)});
965981
}
966982

967983
if (PGOAnalyses)

‎llvm/lib/ObjectYAML/ELFEmitter.cpp‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1465,13 +1465,19 @@ void ELFState<ELFT>::writeSectionContent(
14651465
for (const auto &[Idx, E] : llvm::enumerate(*Section.Entries)) {
14661466
// Write version and feature values.
14671467
if (Section.Type == llvm::ELF::SHT_LLVM_BB_ADDR_MAP) {
1468-
if (E.Version > 4)
1468+
if (E.Version > 5)
14691469
WithColor::warning() << "unsupported SHT_LLVM_BB_ADDR_MAP version: "
14701470
<< static_cast<int>(E.Version)
14711471
<< "; encoding using the most recent version";
14721472
CBA.write(E.Version);
1473-
CBA.write(E.Feature);
1474-
SHeader.sh_size += 2;
1473+
SHeader.sh_size += 1;
1474+
if (E.Version < 5) {
1475+
CBA.write(static_cast<uint8_t>(E.Feature));
1476+
SHeader.sh_size += 1;
1477+
} else {
1478+
CBA.write<uint16_t>(E.Feature, ELFT::Endianness);
1479+
SHeader.sh_size += 2;
1480+
}
14751481
}
14761482
auto FeatureOrErr = llvm::object::BBAddrMap::Features::decode(E.Feature);
14771483
bool MultiBBRangeFeatureEnabled = false;
@@ -1556,11 +1562,15 @@ void ELFState<ELFT>::writeSectionContent(
15561562
for (const auto &PGOBBE : PGOBBEntries) {
15571563
if (PGOBBE.BBFreq)
15581564
SHeader.sh_size += CBA.writeULEB128(*PGOBBE.BBFreq);
1565+
if (FeatureOrErr->PostLinkCfg || PGOBBE.PostLinkBBFreq.has_value())
1566+
SHeader.sh_size += CBA.writeULEB128(PGOBBE.PostLinkBBFreq.value_or(0));
15591567
if (PGOBBE.Successors) {
15601568
SHeader.sh_size += CBA.writeULEB128(PGOBBE.Successors->size());
1561-
for (const auto &[ID, BrProb] : *PGOBBE.Successors) {
1569+
for (const auto &[ID, BrProb, PostLinkBrFreq] : *PGOBBE.Successors) {
15621570
SHeader.sh_size += CBA.writeULEB128(ID);
15631571
SHeader.sh_size += CBA.writeULEB128(BrProb);
1572+
if (FeatureOrErr->PostLinkCfg || PostLinkBrFreq.has_value())
1573+
SHeader.sh_size += CBA.writeULEB128(PostLinkBrFreq.value_or(0));
15641574
}
15651575
}
15661576
}

‎llvm/lib/ObjectYAML/ELFYAML.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1886,7 +1886,7 @@ void MappingTraits<ELFYAML::BBAddrMapEntry>::mapping(
18861886
IO &IO, ELFYAML::BBAddrMapEntry &E) {
18871887
assert(IO.getContext() && "The IO context is not initialized");
18881888
IO.mapRequired("Version", E.Version);
1889-
IO.mapOptional("Feature", E.Feature, Hex8(0));
1889+
IO.mapOptional("Feature", E.Feature, Hex16(0));
18901890
IO.mapOptional("NumBBRanges", E.NumBBRanges);
18911891
IO.mapOptional("BBRanges", E.BBRanges);
18921892
}
@@ -1920,6 +1920,7 @@ void MappingTraits<ELFYAML::PGOAnalysisMapEntry::PGOBBEntry>::mapping(
19201920
IO &IO, ELFYAML::PGOAnalysisMapEntry::PGOBBEntry &E) {
19211921
assert(IO.getContext() && "The IO context is not initialized");
19221922
IO.mapOptional("BBFreq", E.BBFreq);
1923+
IO.mapOptional("PostLinkBBFreq", E.PostLinkBBFreq);
19231924
IO.mapOptional("Successors", E.Successors);
19241925
}
19251926

@@ -1929,6 +1930,7 @@ void MappingTraits<ELFYAML::PGOAnalysisMapEntry::PGOBBEntry::SuccessorEntry>::
19291930
assert(IO.getContext() && "The IO context is not initialized");
19301931
IO.mapRequired("ID", E.ID);
19311932
IO.mapRequired("BrProb", E.BrProb);
1933+
IO.mapOptional("PostLinkBrFreq", E.PostLinkBrFreq);
19321934
}
19331935

19341936
void MappingTraits<ELFYAML::GnuHashHeader>::mapping(IO &IO,

‎llvm/test/tools/llvm-readobj/ELF/bb-addr-map-feature-warning.test‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
## This test checks that we output a warning when the specified version is too old to support the given features.
2+
3+
# RUN: yaml2obj %s -o %t
4+
# RUN: llvm-readobj --bb-addr-map %t 2>&1 | FileCheck -DFILE=%t %s
5+
6+
--- !ELF
7+
FileHeader:
8+
Class: ELFCLASS64
9+
Data: ELFDATA2LSB
10+
Type: ET_EXEC
11+
12+
# CHECK: BBAddrMap [
13+
# CHECK-NEXT: warning: '[[FILE]]': unable to dump SHT_LLVM_BB_ADDR_MAP section with index 1: version should be >= 3 for SHT_LLVM_BB_ADDR_MAP when callsite offsets feature is enabled: version = 2 feature = 32
14+
Sections:
15+
- Name: '.llvm_bb_addr_map (1)'
16+
Type: SHT_LLVM_BB_ADDR_MAP
17+
Entries:
18+
- Version: 2
19+
Feature: 0x20
20+
21+
# CHECK: BBAddrMap [
22+
# CHECK-NEXT: warning: '[[FILE]]': unable to dump SHT_LLVM_BB_ADDR_MAP section with index 2: version should be >= 4 for SHT_LLVM_BB_ADDR_MAP when basic block hash feature is enabled: version = 3 feature = 64
23+
24+
- Name: '.llvm_bb_addr_map (2)'
25+
Type: SHT_LLVM_BB_ADDR_MAP
26+
Entries:
27+
- Version: 3
28+
Feature: 0x40
29+
30+
# CHECK: BBAddrMap [
31+
# CHECK-NEXT: warning: '[[FILE]]': unable to dump SHT_LLVM_BB_ADDR_MAP section with index 3: version should be >= 5 for SHT_LLVM_BB_ADDR_MAP when post link cfg feature is enabled: version = 4 feature = 128
32+
33+
- Name: '.llvm_bb_addr_map (3)'
34+
Type: SHT_LLVM_BB_ADDR_MAP
35+
Entries:
36+
- Version: 4
37+
Feature: 0x80

‎llvm/test/tools/llvm-readobj/ELF/bb-addr-map-pgo-analysis-map.test‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
## Check that a malformed section can be handled.
1717
# RUN: yaml2obj %s -DBITS=32 -DSIZE=24 -o %t2.o
18-
# RUN: llvm-readobj %t2.o --bb-addr-map 2>&1 | FileCheck --match-full-lines %s -DOFFSET=0x00000018 -DFILE=%t2.o --check-prefix=TRUNCATED
18+
# RUN: llvm-readobj %t2.o --bb-addr-map 2>&1 | FileCheck --match-full-lines %s -DOFFSET=0x00000015 -DFILE=%t2.o --check-prefix=TRUNCATED
1919

2020
## Check that missing features can be handled.
2121
# RUN: yaml2obj %s -DBITS=32 -DFEATURE=0x2 -o %t3.o
@@ -59,17 +59,20 @@
5959
# CHECK-NEXT: {
6060
# RAW-NEXT: Frequency: 100
6161
# PRETTY-NEXT: Frequency: 1.0
62+
# CHECK-NEXT: PostLink Frequency: 10
6263
# CHECK-NEXT: Successors [
6364
# CHECK-NEXT: {
6465
# CHECK-NEXT: ID: 2
6566
# RAW-NEXT: Probability: 0x80000000
6667
# PRETTY-NEXT: Probability: 0x80000000 / 0x80000000 = 100.00%
68+
# CHECK-NEXT: PostLink Probability: 7
6769
# CHECK-NEXT: }
6870
# CHECK-NEXT: ]
6971
# CHECK-NEXT: }
7072
# CHECK-NEXT: {
7173
# RAW-NEXT: Frequency: 100
7274
# PRETTY-NEXT: Frequency: 1.0
75+
# CHECK-NEXT: PostLink Frequency: 0
7376
# CHECK-NEXT: Successors [
7477
# CHECK-NEXT: ]
7578
# CHECK-NEXT: }
@@ -172,8 +175,8 @@ Sections:
172175
ShSize: [[SIZE=<none>]]
173176
Link: .text
174177
Entries:
175-
- Version: 2
176-
Feature: 0x7
178+
- Version: 5
179+
Feature: 0x87
177180
BBRanges:
178181
- BaseAddress: [[ADDR=0x11111]]
179182
BBEntries:
@@ -197,10 +200,12 @@ Sections:
197200
PGOAnalyses:
198201
- FuncEntryCount: 100
199202
PGOBBEntries:
200-
- BBFreq: 100
203+
- BBFreq: 100
204+
PostLinkBBFreq: 10
201205
Successors:
202-
- ID: 2
203-
BrProb: 0x80000000
206+
- ID: 2
207+
BrProb: 0x80000000
208+
PostLinkBrFreq: 7
204209
- BBFreq: 100
205210
Successors: []
206211
- FuncEntryCount: 8888

0 commit comments

Comments
 (0)