From d15e3af52de12cb230f3888fa03a027879ffa9d8 Mon Sep 17 00:00:00 2001 From: wudexin Date: Wed, 24 Sep 2025 18:42:21 +0800 Subject: [PATCH] Adding Matching and Inference Functionality to Propeller-PR2: Compute basic block hash and emit to ELF. Co-authored-by: lifengxiang1025 Co-authored-by: zcfh --- .../llvm/CodeGen/MachineBlockHashInfo.h | 121 ++++++++++++++++++ llvm/include/llvm/CodeGen/Passes.h | 3 + llvm/include/llvm/InitializePasses.h | 1 + llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 14 +- llvm/lib/CodeGen/CMakeLists.txt | 1 + llvm/lib/CodeGen/MachineBlockHashInfo.cpp | 114 +++++++++++++++++ llvm/lib/CodeGen/TargetPassConfig.cpp | 8 ++ ...sic-block-address-map-with-emit-bb-hash.ll | 94 ++++++++++++++ 8 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 llvm/include/llvm/CodeGen/MachineBlockHashInfo.h create mode 100644 llvm/lib/CodeGen/MachineBlockHashInfo.cpp create mode 100644 llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll diff --git a/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h new file mode 100644 index 0000000000000..aa460674ec0e7 --- /dev/null +++ b/llvm/include/llvm/CodeGen/MachineBlockHashInfo.h @@ -0,0 +1,121 @@ +//===- llvm/CodeGen/MachineBlockHashInfo.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Compute the hashes of basic blocks. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H +#define LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H + +#include "llvm/CodeGen/MachineFunctionPass.h" + +namespace llvm { + +/// An object wrapping several components of a basic block hash. The combined +/// (blended) hash is represented and stored as one uint64_t, while individual +/// components are of smaller size (e.g., uint16_t or uint8_t). +struct BlendedBlockHash { +public: + explicit BlendedBlockHash(uint16_t Offset, uint16_t OpcodeHash, + uint16_t InstrHash, uint16_t NeighborHash) + : Offset(Offset), OpcodeHash(OpcodeHash), InstrHash(InstrHash), + NeighborHash(NeighborHash) {} + + explicit BlendedBlockHash(uint64_t CombinedHash) { + Offset = CombinedHash & 0xffff; + CombinedHash >>= 16; + OpcodeHash = CombinedHash & 0xffff; + CombinedHash >>= 16; + InstrHash = CombinedHash & 0xffff; + CombinedHash >>= 16; + NeighborHash = CombinedHash & 0xffff; + } + + /// Combine the blended hash into uint64_t. + uint64_t combine() const { + + uint64_t Hash = 0; + + Hash |= uint64_t(NeighborHash); + Hash <<= 16; + + Hash |= uint64_t(InstrHash); + Hash <<= 16; + + Hash |= uint64_t(OpcodeHash); + Hash <<= 16; + + Hash |= uint64_t(Offset); + + return Hash; + } + + /// Compute a distance between two given blended hashes. The smaller the + /// distance, the more similar two blocks are. For identical basic blocks, + /// the distance is zero. + /// Since OpcodeHash is highly stable, we consider a match good only if + /// the OpcodeHashes are identical. Mismatched OpcodeHashes lead to low + /// matching accuracy, and poor matches undermine the quality of final + /// inference. Notably, during inference, we also consider the matching + /// ratio of basic blocks (BBs). For MachineFunctions with a low matching + /// ratio, we directly skip optimization to reduce the impact of + /// mismatches—this ensures even very poor profiles won’t cause negative + /// optimization. + /// In the context of matching, we consider NeighborHash to be more + /// important. This is especially true when accounting for inlining + /// scenarios, where the position of a basic block (BB) in the control + /// flow graph (CFG) is more critical. Therefore, the matching results + /// from NeighborHash carry greater significance. + uint64_t distance(const BlendedBlockHash &BBH) const { + assert(OpcodeHash == BBH.OpcodeHash && + "incorrect blended hash distance computation"); + uint64_t Dist = 0; + // Account for NeighborHash + Dist += NeighborHash == BBH.NeighborHash ? 0 : 1; + Dist <<= 16; + // Account for InstrHash + Dist += InstrHash == BBH.InstrHash ? 0 : 1; + Dist <<= 16; + // Account for Offset + Dist += (Offset >= BBH.Offset ? Offset - BBH.Offset : BBH.Offset - Offset); + return Dist; + } + +private: + /// The offset of the basic block from the function start. + uint16_t Offset{0}; + /// Hash of the basic block instructions, excluding operands. + uint16_t OpcodeHash{0}; + /// Hash of the basic block instructions, including opcodes and + /// operands. + uint16_t InstrHash{0}; + /// OpcodeHash of the basic block together with OpcodeHashes of its + /// successors and predecessors. + uint16_t NeighborHash{0}; +}; + +class MachineBlockHashInfo : public MachineFunctionPass { + DenseMap MBBHashInfo; + +public: + static char ID; + MachineBlockHashInfo(); + + StringRef getPassName() const override { return "Basic Block Hash Compute"; } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + bool runOnMachineFunction(MachineFunction &F) override; + + uint64_t getMBBHash(const MachineBasicBlock &MBB); +}; + +} // end namespace llvm + +#endif // LLVM_CODEGEN_MACHINEBLOCKHASHINFO_H diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h index 272b4acf950c5..8482dbbc2a368 100644 --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -69,6 +69,9 @@ LLVM_ABI MachineFunctionPass *createBasicBlockSectionsPass(); LLVM_ABI MachineFunctionPass *createBasicBlockPathCloningPass(); +/// createMachineBlockHashInfoPass - This pass computes basic block hashes. +LLVM_ABI MachineFunctionPass *createMachineBlockHashInfoPass(); + /// createMachineFunctionSplitterPass - This pass splits machine functions /// using profile information. LLVM_ABI MachineFunctionPass *createMachineFunctionSplitterPass(); diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index cd774e7888e64..024397dab236c 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -62,6 +62,7 @@ LLVM_ABI void initializeBasicBlockSectionsPass(PassRegistry &); LLVM_ABI void initializeBarrierNoopPass(PassRegistry &); LLVM_ABI void initializeBasicAAWrapperPassPass(PassRegistry &); LLVM_ABI void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry &); +LLVM_ABI void initializeMachineBlockHashInfoPass(PassRegistry &); LLVM_ABI void initializeBranchFolderLegacyPass(PassRegistry &); LLVM_ABI void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &); LLVM_ABI void initializeBranchRelaxationLegacyPass(PassRegistry &); diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index e2af0c5925248..1b5a986c1bff8 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -41,6 +41,7 @@ #include "llvm/CodeGen/GCMetadataPrinter.h" #include "llvm/CodeGen/LazyMachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineBlockHashInfo.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineDominators.h" @@ -184,6 +185,8 @@ static cl::opt PrintLatency( cl::desc("Print instruction latencies as verbose asm comments"), cl::Hidden, cl::init(false)); +extern cl::opt EmitBBHash; + STATISTIC(EmittedInsts, "Number of machine instrs printed"); char AsmPrinter::ID = 0; @@ -474,6 +477,8 @@ void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); AU.addRequired(); AU.addRequired(); + if (EmitBBHash) + AU.addRequired(); } bool AsmPrinter::doInitialization(Module &M) { @@ -1440,7 +1445,7 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges, MF.hasBBSections() && NumMBBSectionRanges > 1, static_cast(BBAddrMapSkipEmitBBEntries), HasCalls, - false}; + EmitBBHash.getValue()}; } void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) { @@ -1499,6 +1504,9 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) { PrevMBBEndSymbol = MBBSymbol; } + auto MBHI = + Features.BBHash ? &getAnalysis() : nullptr; + if (!Features.OmitBBEntries) { OutStreamer->AddComment("BB id"); // Emit the BB ID for this basic block. @@ -1526,6 +1534,10 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) { emitLabelDifferenceAsULEB128(MBB.getEndSymbol(), CurrentLabel); // Emit the Metadata. OutStreamer->emitULEB128IntValue(getBBAddrMapMetadata(MBB)); + // Emit the Hash. + if (MBHI) { + OutStreamer->emitInt64(MBHI->getMBBHash(MBB)); + } } PrevMBBEndSymbol = MBB.getEndSymbol(); } diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt index b6872605e22aa..4373c5397a3c6 100644 --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -108,6 +108,7 @@ add_llvm_component_library(LLVMCodeGen LowerEmuTLS.cpp MachineBasicBlock.cpp MachineBlockFrequencyInfo.cpp + MachineBlockHashInfo.cpp MachineBlockPlacement.cpp MachineBranchProbabilityInfo.cpp MachineCFGPrinter.cpp diff --git a/llvm/lib/CodeGen/MachineBlockHashInfo.cpp b/llvm/lib/CodeGen/MachineBlockHashInfo.cpp new file mode 100644 index 0000000000000..740672b90d070 --- /dev/null +++ b/llvm/lib/CodeGen/MachineBlockHashInfo.cpp @@ -0,0 +1,114 @@ +//===- llvm/CodeGen/MachineBlockHashInfo.cpp---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Compute the hashes of basic blocks. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineBlockHashInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/InitializePasses.h" +#include "llvm/Target/TargetMachine.h" + +using namespace llvm; + +uint64_t hashBlock(const MachineBasicBlock &MBB, bool HashOperands) { + uint64_t Hash = 0; + for (const MachineInstr &MI : MBB) { + if (MI.isMetaInstruction() || MI.isTerminator()) + continue; + Hash = hashing::detail::hash_16_bytes(Hash, MI.getOpcode()); + if (HashOperands) { + for (unsigned i = 0; i < MI.getNumOperands(); i++) { + Hash = + hashing::detail::hash_16_bytes(Hash, hash_value(MI.getOperand(i))); + } + } + } + return Hash; +} + +/// Fold a 64-bit integer to a 16-bit one. +uint16_t fold_64_to_16(const uint64_t Value) { + uint16_t Res = static_cast(Value); + Res ^= static_cast(Value >> 16); + Res ^= static_cast(Value >> 32); + Res ^= static_cast(Value >> 48); + return Res; +} + +INITIALIZE_PASS(MachineBlockHashInfo, "machine-block-hash", + "Machine Block Hash Analysis", true, true) + +char MachineBlockHashInfo::ID = 0; + +MachineBlockHashInfo::MachineBlockHashInfo() : MachineFunctionPass(ID) { + initializeMachineBlockHashInfoPass(*PassRegistry::getPassRegistry()); +} + +void MachineBlockHashInfo::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesAll(); + MachineFunctionPass::getAnalysisUsage(AU); +} + +struct CollectHashInfo { + uint64_t Offset; + uint64_t OpcodeHash; + uint64_t InstrHash; + uint64_t NeighborHash; +}; + +bool MachineBlockHashInfo::runOnMachineFunction(MachineFunction &F) { + DenseMap HashInfos; + uint16_t Offset = 0; + // Initialize hash components + for (const MachineBasicBlock &MBB : F) { + // offset of the machine basic block + HashInfos[&MBB].Offset = Offset; + Offset += MBB.size(); + // Hashing opcodes + HashInfos[&MBB].OpcodeHash = hashBlock(MBB, /*HashOperands=*/false); + // Hash complete instructions + HashInfos[&MBB].InstrHash = hashBlock(MBB, /*HashOperands=*/true); + } + + // Initialize neighbor hash + for (const MachineBasicBlock &MBB : F) { + uint64_t Hash = HashInfos[&MBB].OpcodeHash; + // Append hashes of successors + for (const MachineBasicBlock *SuccMBB : MBB.successors()) { + uint64_t SuccHash = HashInfos[SuccMBB].OpcodeHash; + Hash = hashing::detail::hash_16_bytes(Hash, SuccHash); + } + // Append hashes of predecessors + for (const MachineBasicBlock *PredMBB : MBB.predecessors()) { + uint64_t PredHash = HashInfos[PredMBB].OpcodeHash; + Hash = hashing::detail::hash_16_bytes(Hash, PredHash); + } + HashInfos[&MBB].NeighborHash = Hash; + } + + // Assign hashes + for (const MachineBasicBlock &MBB : F) { + BlendedBlockHash BlendedHash(fold_64_to_16(HashInfos[&MBB].Offset), + fold_64_to_16(HashInfos[&MBB].OpcodeHash), + fold_64_to_16(HashInfos[&MBB].InstrHash), + fold_64_to_16(HashInfos[&MBB].NeighborHash)); + MBBHashInfo[&MBB] = BlendedHash.combine(); + } + + return false; +} + +uint64_t MachineBlockHashInfo::getMBBHash(const MachineBasicBlock &MBB) { + return MBBHashInfo[&MBB]; +} + +MachineFunctionPass *llvm::createMachineBlockHashInfoPass() { + return new MachineBlockHashInfo(); +} diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index b6169e6c4dc34..10b723887b21f 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -272,6 +272,12 @@ static cl::opt cl::desc("Split static data sections into hot and cold " "sections using profile information")); +cl::opt EmitBBHash( + "emit-bb-hash", + cl::desc( + "Emit the hash of basic block in the SHT_LLVM_BB_ADDR_MAP section."), + cl::init(false), cl::Optional); + /// Allow standard passes to be disabled by command line options. This supports /// simple binary flags that either suppress the pass or do nothing. /// i.e. -disable-mypass=false has no effect. @@ -1281,6 +1287,8 @@ void TargetPassConfig::addMachinePasses() { // address map (or both). if (TM->getBBSectionsType() != llvm::BasicBlockSection::None || TM->Options.BBAddrMap) { + if (EmitBBHash) + addPass(llvm::createMachineBlockHashInfoPass()); if (TM->getBBSectionsType() == llvm::BasicBlockSection::List) { addPass(llvm::createBasicBlockSectionsProfileReaderWrapperPass( TM->getBBSectionsFuncListBuf())); diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll new file mode 100644 index 0000000000000..b89f783adbf9b --- /dev/null +++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll @@ -0,0 +1,94 @@ +; Check the basic block sections labels option works when used along with -emit-bb-hash. +; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -emit-bb-hash | FileCheck %s --check-prefixes=CHECK,UNIQ + +define void @_Z3bazb(i1 zeroext, i1 zeroext) personality ptr @__gxx_personality_v0 { + br i1 %0, label %3, label %8 + +3: + %4 = invoke i32 @_Z3barv() + to label %8 unwind label %6 + br label %10 + +6: + landingpad { ptr, i32 } + catch ptr null + br label %12 + +8: + %9 = call i32 @_Z3foov() + br i1 %1, label %12, label %10 + +10: + %11 = select i1 %1, ptr blockaddress(@_Z3bazb, %3), ptr blockaddress(@_Z3bazb, %12) ; [#uses=1] + indirectbr ptr %11, [label %3, label %12] + +12: + ret void +} + +declare i32 @_Z3barv() #1 + +declare i32 @_Z3foov() #1 + +declare i32 @__gxx_personality_v0(...) + +; UNIQ: .section .text._Z3bazb,"ax",@progbits{{$}} +; NOUNIQ: .section .text,"ax",@progbits,unique,1 +; CHECK-LABEL: _Z3bazb: +; CHECK-LABEL: .Lfunc_begin0: +; CHECK-LABEL: .LBB_END0_0: +; CHECK-LABEL: .LBB0_1: +; CHECK-LABEL: .LBB0_1_CS0: +; CHECK-LABEL: .LBB_END0_1: +; CHECK-LABEL: .LBB0_2: +; CHECK-LABEL: .LBB0_2_CS0: +; CHECK-LABEL: .LBB_END0_2: +; CHECK-LABEL: .LBB0_3: +; CHECK-LABEL: .LBB_END0_3: +; CHECK-LABEL: .Lfunc_end0: + +; UNIQ: .section .llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3bazb{{$}} +;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section. +; NOUNIQ: .section .llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text,unique,1 +; CHECK-NEXT: .byte 3 # version +; CHECK-NEXT: .byte 96 # feature +; CHECK-NEXT: .quad .Lfunc_begin0 # function address +; CHECK-NEXT: .byte 6 # number of basic blocks +; CHECK-NEXT: .byte 0 # BB id +; CHECK-NEXT: .uleb128 .Lfunc_begin0-.Lfunc_begin0 +; CHECK-NEXT: .byte 0 # number of callsites +; CHECK-NEXT: .uleb128 .LBB_END0_0-.Lfunc_begin0 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .quad {{-?[0-9]+}} +; CHECK-NEXT: .byte 1 # BB id +; CHECK-NEXT: .uleb128 .LBB0_1-.LBB_END0_0 +; CHECK-NEXT: .byte 1 # number of callsites +; CHECK-NEXT: .uleb128 .LBB0_1_CS0-.LBB0_1 +; CHECK-NEXT: .uleb128 .LBB_END0_1-.LBB0_1_CS0 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .quad {{-?[0-9]+}} +; CHECK-NEXT: .byte 3 # BB id +; CHECK-NEXT: .uleb128 .LBB0_2-.LBB_END0_1 +; CHECK-NEXT: .byte 1 # number of callsites +; CHECK-NEXT: .uleb128 .LBB0_2_CS0-.LBB0_2 +; CHECK-NEXT: .uleb128 .LBB_END0_2-.LBB0_2_CS0 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .quad {{-?[0-9]+}} +; CHECK-NEXT: .byte 4 # BB id +; CHECK-NEXT: .uleb128 .LBB0_3-.LBB_END0_2 +; CHECK-NEXT: .byte 0 # number of callsites +; CHECK-NEXT: .uleb128 .LBB_END0_3-.LBB0_3 +; CHECK-NEXT: .byte 16 +; CHECK-NEXT: .quad {{-?[0-9]+}} +; CHECK-NEXT: .byte 5 # BB id +; CHECK-NEXT: .uleb128 .LBB0_4-.LBB_END0_3 +; CHECK-NEXT: .byte 0 # number of callsites +; CHECK-NEXT: .uleb128 .LBB_END0_4-.LBB0_4 +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .quad {{-?[0-9]+}} +; CHECK-NEXT: .byte 2 # BB id +; CHECK-NEXT: .uleb128 .LBB0_5-.LBB_END0_4 +; CHECK-NEXT: .byte 0 # number of callsites +; CHECK-NEXT: .uleb128 .LBB_END0_5-.LBB0_5 +; CHECK-NEXT: .byte 5 +; CHECK-NEXT: .quad {{-?[0-9]+}}