Skip to content

[AMDGPU] Refine GCNHazardRecognizer hasHazard() #138841

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 24, 2025
Merged

[AMDGPU] Refine GCNHazardRecognizer hasHazard() #138841

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0230c8e
[AMDGPU] Refine GCNHazardRecognizer hasHazard()
perlfu May 6, 2025
2a18fbe
- Rework to use unified store of states
perlfu May 18, 2025
9532454
- Fix use of llvm::find_if
perlfu May 19, 2025
61a3e48
- Rewrite most of custom implementation to DenseMap with traits
perlfu Sep 9, 2025
5a736bd
- Address reviewer comments
perlfu Sep 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 106 additions & 31 deletions llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -443,40 +443,101 @@ using GetNumWaitStatesFn = function_ref<unsigned int(const MachineInstr &)>;
// Search for a hazard in a block and its predecessors.
template <typename StateT>
static bool
hasHazard(StateT State,
hasHazard(StateT InitialState,
function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
function_ref<void(StateT &, const MachineInstr &)> UpdateState,
Comment on lines 447 to 448
Copy link
Contributor

@jayfoad jayfoad Sep 4, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not something to address in the current patch, but I think this IsHazard/UpdateState interface is confusing because IsHazard also updates the state. So technically I think there is no need for the separate UpdateState function, it could just become part of what IsHazard does before returning NoHazardFound.

const MachineBasicBlock *MBB,
MachineBasicBlock::const_reverse_instr_iterator I,
DenseSet<const MachineBasicBlock *> &Visited) {
for (auto E = MBB->instr_rend(); I != E; ++I) {
// No need to look at parent BUNDLE instructions.
if (I->isBundle())
continue;

switch (IsHazard(State, *I)) {
case HazardFound:
return true;
case HazardExpired:
return false;
default:
// Continue search
break;
const MachineBasicBlock *InitialMBB,
MachineBasicBlock::const_reverse_instr_iterator InitialI) {
struct StateMapKey {
SmallVectorImpl<StateT> *States;
unsigned Idx;
static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
return LHS.States == RHS.States && LHS.Idx == RHS.Idx;
}
};
struct StateMapKeyTraits : DenseMapInfo<StateMapKey> {
static inline StateMapKey getEmptyKey() {
return {static_cast<SmallVectorImpl<StateT> *>(
DenseMapInfo<void *>::getEmptyKey()),
DenseMapInfo<unsigned>::getEmptyKey()};
}
static inline StateMapKey getTombstoneKey() {
return {static_cast<SmallVectorImpl<StateT> *>(
DenseMapInfo<void *>::getTombstoneKey()),
DenseMapInfo<unsigned>::getTombstoneKey()};
}
static unsigned getHashValue(const StateMapKey &Key) {
return StateT::getHashValue((*Key.States)[Key.Idx]);
}
static unsigned getHashValue(const StateT &State) {
return StateT::getHashValue(State);
}
static bool isEqual(const StateMapKey &LHS, const StateMapKey &RHS) {
const auto EKey = getEmptyKey();
const auto TKey = getTombstoneKey();
if (StateMapKey::isEqual(LHS, EKey) || StateMapKey::isEqual(RHS, EKey) ||
StateMapKey::isEqual(LHS, TKey) || StateMapKey::isEqual(RHS, TKey))
return StateMapKey::isEqual(LHS, RHS);
return StateT::isEqual((*LHS.States)[LHS.Idx], (*RHS.States)[RHS.Idx]);
}
static bool isEqual(const StateT &LHS, const StateMapKey &RHS) {
if (StateMapKey::isEqual(RHS, getEmptyKey()) ||
StateMapKey::isEqual(RHS, getTombstoneKey()))
return false;
return StateT::isEqual(LHS, (*RHS.States)[RHS.Idx]);
}
};

if (I->isInlineAsm() || I->isMetaInstruction())
continue;
SmallDenseMap<StateMapKey, unsigned, 8, StateMapKeyTraits> StateMap;
SmallVector<StateT, 8> States;

UpdateState(State, *I);
}
MachineBasicBlock::const_reverse_instr_iterator I = InitialI;
const MachineBasicBlock *MBB = InitialMBB;
StateT State = InitialState;

for (MachineBasicBlock *Pred : MBB->predecessors()) {
if (!Visited.insert(Pred).second)
continue;
SmallSetVector<std::pair<const MachineBasicBlock *, unsigned>, 16> Worklist;
unsigned WorkIdx = 0;
for (;;) {
bool Expired = false;
for (auto E = MBB->instr_rend(); I != E; ++I) {
// No need to look at parent BUNDLE instructions.
if (I->isBundle())
continue;

if (hasHazard(State, IsHazard, UpdateState, Pred, Pred->instr_rbegin(),
Visited))
return true;
auto Result = IsHazard(State, *I);
if (Result == HazardFound)
return true;
if (Result == HazardExpired) {
Expired = true;
break;
}

if (I->isInlineAsm() || I->isMetaInstruction())
continue;

UpdateState(State, *I);
}

if (!Expired) {
unsigned StateIdx = States.size();
StateMapKey Key = {&States, StateIdx};
auto Insertion = StateMap.insert_as(std::pair(Key, StateIdx), State);
if (Insertion.second) {
States.emplace_back(State);
} else {
StateIdx = Insertion.first->second;
}
for (MachineBasicBlock *Pred : MBB->predecessors())
Worklist.insert(std::pair(Pred, StateIdx));
}

if (WorkIdx == Worklist.size())
break;

unsigned StateIdx;
std::tie(MBB, StateIdx) = Worklist[WorkIdx++];
State = States[StateIdx];
I = MBB->instr_rbegin();
}

return false;
Expand Down Expand Up @@ -1641,6 +1702,15 @@ bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
SmallDenseMap<Register, int, 4> DefPos;
int ExecPos = std::numeric_limits<int>::max();
int VALUs = 0;

static unsigned getHashValue(const StateType &State) {
return hash_combine(State.ExecPos, State.VALUs,
hash_combine_range(State.DefPos));
}
static bool isEqual(const StateType &LHS, const StateType &RHS) {
return LHS.DefPos == RHS.DefPos && LHS.ExecPos == RHS.ExecPos &&
LHS.VALUs == RHS.VALUs;
}
};

StateType State;
Expand Down Expand Up @@ -1735,9 +1805,8 @@ bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
State.VALUs += 1;
};

DenseSet<const MachineBasicBlock *> Visited;
if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
std::next(MI->getReverseIterator()), Visited))
std::next(MI->getReverseIterator())))
return false;

BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
Expand Down Expand Up @@ -1778,6 +1847,13 @@ bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
struct StateType {
int VALUs = 0;
int TRANS = 0;

static unsigned getHashValue(const StateType &State) {
return hash_combine(State.VALUs, State.TRANS);
}
static bool isEqual(const StateType &LHS, const StateType &RHS) {
return LHS.VALUs == RHS.VALUs && LHS.TRANS == RHS.TRANS;
}
};

StateType State;
Expand Down Expand Up @@ -1813,9 +1889,8 @@ bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
State.TRANS += 1;
};

DenseSet<const MachineBasicBlock *> Visited;
if (!hasHazard<StateType>(State, IsHazardFn, UpdateStateFn, MI->getParent(),
std::next(MI->getReverseIterator()), Visited))
std::next(MI->getReverseIterator())))
return false;

// Hazard is observed - insert a wait on va_dst counter to ensure hazard is
Expand Down
Loading