Skip to content

HDFS-15551. Tiny Improve for DeadNode detector #2265

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 11, 2020
Merged

HDFS-15551. Tiny Improve for DeadNode detector #2265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ec130e9
DeadNode detector's tiny improve
imbajin Sep 1, 2020
1d3aa75
adjust LOG level
imbajin Sep 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ...-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,10 +181,13 @@ private boolean isPeriodicRefreshEnabled() {
private byte[] oneByteBuf; // used for 'int read()'

protected void addToLocalDeadNodes(DatanodeInfo dnInfo) {
DFSClient.LOG.debug("Add {} to local dead nodes, previously was {}.",
dnInfo, deadNodes);
deadNodes.put(dnInfo, dnInfo);
}

protected void removeFromLocalDeadNodes(DatanodeInfo dnInfo) {
DFSClient.LOG.debug("Remove {} from local dead nodes.", dnInfo);
deadNodes.remove(dnInfo);
}

Expand Down
11 changes: 6 additions & 5 deletions ...dfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ void startProbeScheduler() {
}

/**
* Prode datanode by probe byte.
* Prode datanode by probe type.
*/
private void scheduleProbe(ProbeType type) {
LOG.debug("Schedule probe datanode for probe type: {}.", type);
Expand Down Expand Up @@ -376,9 +376,8 @@ public DatanodeLocalInfo call() throws Exception {
} catch (Exception e) {
LOG.error("Probe failed, datanode: {}, type: {}.", datanodeInfo, type,
e);
deadNodeDetector.probeCallBack(this, false);
}

deadNodeDetector.probeCallBack(this, false);
}
}

Expand All @@ -402,7 +401,7 @@ private void probeCallBack(Probe probe, boolean success) {
}
} else {
if (probe.getType() == ProbeType.CHECK_SUSPECT) {
LOG.info("Add the node to dead node list: {}.",
LOG.warn("Probe failed, add suspect node to dead node list: {}.",
probe.getDatanodeInfo());
addToDead(probe.getDatanodeInfo());
}
Expand All @@ -415,11 +414,12 @@ private void probeCallBack(Probe probe, boolean success) {
private void checkDeadNodes() {
Set<DatanodeInfo> datanodeInfos = clearAndGetDetectedDeadNodes();
for (DatanodeInfo datanodeInfo : datanodeInfos) {
LOG.debug("Add dead node to check: {}.", datanodeInfo);
if (!deadNodesProbeQueue.offer(datanodeInfo)) {
LOG.debug("Skip to add dead node {} to check " +
"since the probe queue is full.", datanodeInfo);
break;
} else {
LOG.debug("Add dead node to check: {}.", datanodeInfo);
}
}
state = State.IDLE;
Expand Down Expand Up @@ -475,6 +475,7 @@ public synchronized void addNodeToDetect(DFSInputStream dfsInputStream,
datanodeInfos.add(datanodeInfo);
}

LOG.debug("Add datanode {} to suspectAndDeadNodes.", datanodeInfo);
addSuspectNodeToDetect(datanodeInfo);
}

Expand Down