From 47dc1d835569d2309f84bada24a0b31330a6948a Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Fri, 27 Oct 2023 16:27:37 +0800 Subject: [PATCH 1/7] HDFS-17242. Make congestion backoff time configurable. --- .../org/apache/hadoop/hdfs/DataStreamer.java | 44 +++++++++++++++---- .../hdfs/client/HdfsClientConfigKeys.java | 8 ++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index d92f5943fd8a2..e4e4088ffae7b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.SUCCESS; import java.io.BufferedOutputStream; +import java.io.Closeable; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -29,6 +30,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; +import java.net.UnknownHostException; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; import java.util.Arrays; @@ -46,6 +48,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.BlockWrite; import org.apache.hadoop.hdfs.client.impl.DfsClientConf; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -151,7 +154,7 @@ void recordFailure(final InvalidEncryptionKeyException e) } } - private class StreamerStreams implements java.io.Closeable { + private class StreamerStreams implements Closeable { private Socket sock = null; private DataOutputStream out = null; private DataInputStream in = null; @@ -528,9 +531,8 @@ boolean doWaitForRestart() { // are congested private final List congestedNodes = new ArrayList<>(); private final Map slowNodeMap = new HashMap<>(); - private static final int CONGESTION_BACKOFF_MEAN_TIME_IN_MS = 5000; - private static final int CONGESTION_BACK_OFF_MAX_TIME_IN_MS = - CONGESTION_BACKOFF_MEAN_TIME_IN_MS * 10; + private int congestionBackOffMeanTimeInMs; + private int congestionBackOffMaxTimeInMs; private int lastCongestionBackoffTime; private int maxPipelineRecoveryRetries; private int markSlowNodeAsBadNodeThreshold; @@ -564,6 +566,32 @@ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block, this.addBlockFlags = flags; this.maxPipelineRecoveryRetries = conf.getMaxPipelineRecoveryRetries(); this.markSlowNodeAsBadNodeThreshold = conf.getMarkSlowNodeAsBadNodeThreshold(); + congestionBackOffMeanTimeInMs = dfsClient.getConfiguration().getInt( + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME, + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT); + congestionBackOffMaxTimeInMs = dfsClient.getConfiguration().getInt( + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT); + if (congestionBackOffMeanTimeInMs <= 0 || congestionBackOffMaxTimeInMs <= 0 || + congestionBackOffMaxTimeInMs < congestionBackOffMeanTimeInMs) { + if (congestionBackOffMeanTimeInMs <= 0) { + LOG.warn("Configuration: {} is not appropriate, use default value: {}", + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME, + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT); + } + if (congestionBackOffMaxTimeInMs <= 0) { + LOG.warn("Configuration: {} is not appropriate, use default value: {}", + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT); + } + if (congestionBackOffMaxTimeInMs < congestionBackOffMeanTimeInMs) { + LOG.warn("Configuration: {} can not less than {}, use their default values.", + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME); + } + congestionBackOffMeanTimeInMs = HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT; + congestionBackOffMaxTimeInMs = HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT; + } } /** @@ -1113,7 +1141,7 @@ boolean shouldWaitForRestart(int index) { InetAddress addr = null; try { addr = InetAddress.getByName(nodes[index].getIpAddr()); - } catch (java.net.UnknownHostException e) { + } catch (UnknownHostException e) { // we are passing an ip address. this should not happen. assert false; } @@ -1998,10 +2026,10 @@ private void backOffIfNecessary() throws InterruptedException { sb.append(' ').append(i); } int range = Math.abs(lastCongestionBackoffTime * 3 - - CONGESTION_BACKOFF_MEAN_TIME_IN_MS); + congestionBackOffMeanTimeInMs); int base = Math.min(lastCongestionBackoffTime * 3, - CONGESTION_BACKOFF_MEAN_TIME_IN_MS); - t = Math.min(CONGESTION_BACK_OFF_MAX_TIME_IN_MS, + congestionBackOffMeanTimeInMs); + t = Math.min(congestionBackOffMaxTimeInMs, (int)(base + Math.random() * range)); lastCongestionBackoffTime = t; sb.append(" are congested. Backing off for ").append(t).append(" ms"); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 2b511bfc2ebee..452f9e0f081c3 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -287,6 +287,14 @@ public interface HdfsClientConfigKeys { "dfs.client.output.stream.uniq.default.key"; String DFS_OUTPUT_STREAM_UNIQ_DEFAULT_KEY_DEFAULT = "DEFAULT"; + String DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME = + "dfs.client.congestion.backoff.mean.time"; + int DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT = 5000; + + String DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME = + "dfs.client.congestion.backoff.max.time"; + int DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT = 50000; + /** * These are deprecated config keys to client code. */ From f0ed203947ecf1f0d1e20efcc367874f095e1362 Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Sun, 29 Oct 2023 19:39:48 +0800 Subject: [PATCH 2/7] fix blanks-eol. --- .../main/java/org/apache/hadoop/hdfs/DataStreamer.java | 8 +++++--- .../apache/hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index e4e4088ffae7b..31ea7db348ea7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -572,7 +572,7 @@ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block, congestionBackOffMaxTimeInMs = dfsClient.getConfiguration().getInt( HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT); - if (congestionBackOffMeanTimeInMs <= 0 || congestionBackOffMaxTimeInMs <= 0 || + if (congestionBackOffMeanTimeInMs <= 0 || congestionBackOffMaxTimeInMs <= 0 || congestionBackOffMaxTimeInMs < congestionBackOffMeanTimeInMs) { if (congestionBackOffMeanTimeInMs <= 0) { LOG.warn("Configuration: {} is not appropriate, use default value: {}", @@ -589,8 +589,10 @@ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block, HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME); } - congestionBackOffMeanTimeInMs = HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT; - congestionBackOffMaxTimeInMs = HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT; + congestionBackOffMeanTimeInMs = + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT; + congestionBackOffMaxTimeInMs = + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 452f9e0f081c3..32b1c52408cbb 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -291,7 +291,7 @@ public interface HdfsClientConfigKeys { "dfs.client.congestion.backoff.mean.time"; int DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT = 5000; - String DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME = + String DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME = "dfs.client.congestion.backoff.max.time"; int DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT = 50000; From 4575027a6db7c4f7ae1cd4d22949a7cda4ce1192 Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Mon, 30 Oct 2023 09:50:40 +0800 Subject: [PATCH 3/7] fix review opinions. --- .../java/org/apache/hadoop/hdfs/DataStreamer.java | 12 +++++------- .../src/main/resources/hdfs-default.xml | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index 31ea7db348ea7..0cc211593ff34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -20,7 +20,6 @@ import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status.SUCCESS; import java.io.BufferedOutputStream; -import java.io.Closeable; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -30,7 +29,6 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; -import java.net.UnknownHostException; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; import java.util.Arrays; @@ -154,7 +152,7 @@ void recordFailure(final InvalidEncryptionKeyException e) } } - private class StreamerStreams implements Closeable { + private class StreamerStreams implements java.io.Closeable { private Socket sock = null; private DataOutputStream out = null; private DataInputStream in = null; @@ -575,17 +573,17 @@ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block, if (congestionBackOffMeanTimeInMs <= 0 || congestionBackOffMaxTimeInMs <= 0 || congestionBackOffMaxTimeInMs < congestionBackOffMeanTimeInMs) { if (congestionBackOffMeanTimeInMs <= 0) { - LOG.warn("Configuration: {} is not appropriate, use default value: {}", + LOG.warn("Configuration: {} is not appropriate, using default value: {}", HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME, HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT); } if (congestionBackOffMaxTimeInMs <= 0) { - LOG.warn("Configuration: {} is not appropriate, use default value: {}", + LOG.warn("Configuration: {} is not appropriate, using default value: {}", HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT); } if (congestionBackOffMaxTimeInMs < congestionBackOffMeanTimeInMs) { - LOG.warn("Configuration: {} can not less than {}, use their default values.", + LOG.warn("Configuration: {} can not less than {}, using their default values.", HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME); } @@ -1143,7 +1141,7 @@ boolean shouldWaitForRestart(int index) { InetAddress addr = null; try { addr = InetAddress.getByName(nodes[index].getIpAddr()); - } catch (UnknownHostException e) { + } catch (java.net.UnknownHostException e) { // we are passing an ip address. this should not happen. assert false; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 3ef433d6a4ba9..1b87e43fb54e6 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -6559,6 +6559,21 @@ If the namespace is DEFAULT, it's best to change this conf to other value. + + dfs.client.congestion.backoff.mean.time + 5000 + + The mean milliseconds which is used to compute client congestion backoff sleep time. + + + + dfs.client.congestion.backoff.max.time + 50000 + + The max milliseconds which is used to + restrict the upper limit backoff sleep time for client. + + dfs.client.rbf.observer.read.enable false From e815dfbbb60c7f1b09ce10a2eb10dec456b275ec Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Tue, 31 Oct 2023 09:52:02 +0800 Subject: [PATCH 4/7] fix failed unit tests. --- .../org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java | 3 ++- .../java/org/apache/hadoop/hdfs/TestDFSOutputStream.java | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 32b1c52408cbb..93e214a403ee9 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -293,7 +293,8 @@ public interface HdfsClientConfigKeys { String DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME = "dfs.client.congestion.backoff.max.time"; - int DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT = 50000; + int DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT = + DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT * 10; /** * These are deprecated config keys to client code. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java index 8b90287e82fe1..0f1b965cc2649 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java @@ -275,6 +275,8 @@ private void runAdjustChunkBoundary( public void testCongestionBackoff() throws IOException { DfsClientConf dfsClientConf = mock(DfsClientConf.class); DFSClient client = mock(DFSClient.class); + Configuration conf = mock(Configuration.class); + when(client.getConfiguration()).thenReturn(conf); when(client.getConf()).thenReturn(dfsClientConf); when(client.getTracer()).thenReturn(FsTracer.get(new Configuration())); client.clientRunning = true; @@ -306,6 +308,8 @@ public void testCongestionBackoff() throws IOException { public void testCongestionAckDelay() { DfsClientConf dfsClientConf = mock(DfsClientConf.class); DFSClient client = mock(DFSClient.class); + Configuration conf = mock(Configuration.class); + when(client.getConfiguration()).thenReturn(conf); when(client.getConf()).thenReturn(dfsClientConf); when(client.getTracer()).thenReturn(FsTracer.get(new Configuration())); client.clientRunning = true; @@ -325,7 +329,7 @@ public void testCongestionAckDelay() { ArrayList congestedNodes = (ArrayList) Whitebox.getInternalState(stream, "congestedNodes"); int backOffMaxTime = (int) - Whitebox.getInternalState(stream, "CONGESTION_BACK_OFF_MAX_TIME_IN_MS"); + Whitebox.getInternalState(stream, "congestionBackOffMaxTimeInMs"); DFSPacket[] packet = new DFSPacket[100]; AtomicBoolean isDelay = new AtomicBoolean(true); From 83e483f1a508ff681bf3bb5198dd9c069ca434b6 Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Fri, 1 Dec 2023 15:52:13 +0800 Subject: [PATCH 5/7] fix some problems. --- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 1b87e43fb54e6..174f7242bfbcf 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -6563,15 +6563,16 @@ dfs.client.congestion.backoff.mean.time 5000 - The mean milliseconds which is used to compute client congestion backoff sleep time. + The mean time in milliseconds which is used to compute + client congestion backoff sleep time. dfs.client.congestion.backoff.max.time 50000 - The max milliseconds which is used to - restrict the upper limit backoff sleep time for client. + The max time in milliseconds which is used to restrict + the upper limit backoff sleep time for client. From ba3df2f384900d4521f13895fe28810543238a75 Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Fri, 1 Dec 2023 16:32:12 +0800 Subject: [PATCH 6/7] fix review suggestions. --- .../java/org/apache/hadoop/hdfs/DataStreamer.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index 0cc211593ff34..68bf3de0e4090 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -572,21 +572,6 @@ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block, HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT); if (congestionBackOffMeanTimeInMs <= 0 || congestionBackOffMaxTimeInMs <= 0 || congestionBackOffMaxTimeInMs < congestionBackOffMeanTimeInMs) { - if (congestionBackOffMeanTimeInMs <= 0) { - LOG.warn("Configuration: {} is not appropriate, using default value: {}", - HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME, - HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT); - } - if (congestionBackOffMaxTimeInMs <= 0) { - LOG.warn("Configuration: {} is not appropriate, using default value: {}", - HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, - HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT); - } - if (congestionBackOffMaxTimeInMs < congestionBackOffMeanTimeInMs) { - LOG.warn("Configuration: {} can not less than {}, using their default values.", - HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, - HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME); - } congestionBackOffMeanTimeInMs = HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT; congestionBackOffMaxTimeInMs = From efefa15c13b687d6bf3a07978486ef6bdcbdf432 Mon Sep 17 00:00:00 2001 From: "zhanghaobo@kanzhun.com" Date: Fri, 1 Dec 2023 20:42:33 +0800 Subject: [PATCH 7/7] divide three individual if checks --- .../org/apache/hadoop/hdfs/DataStreamer.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java index 68bf3de0e4090..04d6d9a565659 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java @@ -570,6 +570,21 @@ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block, congestionBackOffMaxTimeInMs = dfsClient.getConfiguration().getInt( HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT); + if (congestionBackOffMeanTimeInMs <= 0) { + LOG.warn("Configuration: {} is not appropriate, using default value: {}", + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME, + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME_DEFAULT); + } + if (congestionBackOffMaxTimeInMs <= 0) { + LOG.warn("Configuration: {} is not appropriate, using default value: {}", + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT); + } + if (congestionBackOffMaxTimeInMs < congestionBackOffMeanTimeInMs) { + LOG.warn("Configuration: {} can not less than {}, using their default values.", + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME, + HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MEAN_TIME); + } if (congestionBackOffMeanTimeInMs <= 0 || congestionBackOffMaxTimeInMs <= 0 || congestionBackOffMaxTimeInMs < congestionBackOffMeanTimeInMs) { congestionBackOffMeanTimeInMs = @@ -577,6 +592,7 @@ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block, congestionBackOffMaxTimeInMs = HdfsClientConfigKeys.DFS_CLIENT_CONGESTION_BACKOFF_MAX_TIME_DEFAULT; } + } /**