From ec130e9ae0975010049695892addae02f26b36f0 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 1 Sep 2020 17:24:57 +0800 Subject: [PATCH 1/2] DeadNode detector's tiny improve 1. add or improve some logs for adding local & global deadnodes 2. logic improve 3. fix typo --- .../java/org/apache/hadoop/hdfs/DFSInputStream.java | 2 ++ .../org/apache/hadoop/hdfs/DeadNodeDetector.java | 13 +++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 0676cf91a7d9e..7e8b75d09ec94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -181,6 +181,8 @@ private boolean isPeriodicRefreshEnabled() { private byte[] oneByteBuf; // used for 'int read()' protected void addToLocalDeadNodes(DatanodeInfo dnInfo) { + DFSClient.LOG.debug("Add {} to local dead nodes, previously was {}", + dnInfo, deadNodes); deadNodes.put(dnInfo, dnInfo); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java index a573e8a22aad3..2c00627e6a1ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java @@ -294,7 +294,7 @@ void startProbeScheduler() { } /** - * Prode datanode by probe byte. + * Prode datanode by probe type. */ private void scheduleProbe(ProbeType type) { LOG.debug("Schedule probe datanode for probe type: {}.", type); @@ -376,9 +376,8 @@ public DatanodeLocalInfo call() throws Exception { } catch (Exception e) { LOG.error("Probe failed, datanode: {}, type: {}.", datanodeInfo, type, e); + deadNodeDetector.probeCallBack(this, false); } - - deadNodeDetector.probeCallBack(this, false); } } @@ -396,13 +395,13 @@ private void probeCallBack(Probe probe, boolean success) { probe.getDatanodeInfo()); removeDeadNode(probe.getDatanodeInfo()); } else if (probe.getType() == ProbeType.CHECK_SUSPECT) { - LOG.debug("Remove the node out from suspect node list: {}.", + LOG.info("Remove the node out from suspect node list: {}.", probe.getDatanodeInfo()); removeNodeFromDeadNodeDetector(probe.getDatanodeInfo()); } } else { if (probe.getType() == ProbeType.CHECK_SUSPECT) { - LOG.info("Add the node to dead node list: {}.", + LOG.warn("Probe failed, add suspect node to dead node list: {}.", probe.getDatanodeInfo()); addToDead(probe.getDatanodeInfo()); } @@ -415,11 +414,12 @@ private void probeCallBack(Probe probe, boolean success) { private void checkDeadNodes() { Set datanodeInfos = clearAndGetDetectedDeadNodes(); for (DatanodeInfo datanodeInfo : datanodeInfos) { - LOG.debug("Add dead node to check: {}.", datanodeInfo); if (!deadNodesProbeQueue.offer(datanodeInfo)) { LOG.debug("Skip to add dead node {} to check " + "since the probe queue is full.", datanodeInfo); break; + } else { + LOG.debug("Add dead node to check: {}.", datanodeInfo); } } state = State.IDLE; @@ -475,6 +475,7 @@ public synchronized void addNodeToDetect(DFSInputStream dfsInputStream, datanodeInfos.add(datanodeInfo); } + LOG.warn("Add datanode {} to suspectAndDeadNodes", datanodeInfo); addSuspectNodeToDetect(datanodeInfo); } From 1d3aa75b7b7856b43f76f81d04d8cb09b83ce1f5 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 4 Sep 2020 18:25:32 +0800 Subject: [PATCH 2/2] adjust LOG level --- .../src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java | 3 ++- .../main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 7e8b75d09ec94..772354a489dfe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -181,12 +181,13 @@ private boolean isPeriodicRefreshEnabled() { private byte[] oneByteBuf; // used for 'int read()' protected void addToLocalDeadNodes(DatanodeInfo dnInfo) { - DFSClient.LOG.debug("Add {} to local dead nodes, previously was {}", + DFSClient.LOG.debug("Add {} to local dead nodes, previously was {}.", dnInfo, deadNodes); deadNodes.put(dnInfo, dnInfo); } protected void removeFromLocalDeadNodes(DatanodeInfo dnInfo) { + DFSClient.LOG.debug("Remove {} from local dead nodes.", dnInfo); deadNodes.remove(dnInfo); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java index 2c00627e6a1ed..aaa12dbc77c01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java @@ -395,7 +395,7 @@ private void probeCallBack(Probe probe, boolean success) { probe.getDatanodeInfo()); removeDeadNode(probe.getDatanodeInfo()); } else if (probe.getType() == ProbeType.CHECK_SUSPECT) { - LOG.info("Remove the node out from suspect node list: {}.", + LOG.debug("Remove the node out from suspect node list: {}.", probe.getDatanodeInfo()); removeNodeFromDeadNodeDetector(probe.getDatanodeInfo()); } @@ -475,7 +475,7 @@ public synchronized void addNodeToDetect(DFSInputStream dfsInputStream, datanodeInfos.add(datanodeInfo); } - LOG.warn("Add datanode {} to suspectAndDeadNodes", datanodeInfo); + LOG.debug("Add datanode {} to suspectAndDeadNodes.", datanodeInfo); addSuspectNodeToDetect(datanodeInfo); }