From a701f7cb6a4b8a2d853daec05145cac2899689f8 Mon Sep 17 00:00:00 2001 From: Abhey Rana Date: Tue, 4 Feb 2025 18:07:23 +0530 Subject: [PATCH 1/2] HDFS-17724: Set recover.lease.on.close.exception as an instance member in the DfsClientConf.java --- .../org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 7 +------ .../apache/hadoop/hdfs/client/impl/DfsClientConf.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 8320cc9a40866..7a59aa7012883 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -73,9 +73,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_DEFAULT; -import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_KEY; - /** * This class supports writing files in striped layout and erasure coded format. * Each stripe contains a sequence of cells. @@ -1200,9 +1197,7 @@ void setClosed() { @Override protected synchronized void closeImpl() throws IOException { - boolean recoverLeaseOnCloseException = dfsClient.getConfiguration() - .getBoolean(RECOVER_LEASE_ON_CLOSE_EXCEPTION_KEY, - RECOVER_LEASE_ON_CLOSE_EXCEPTION_DEFAULT); + boolean recoverLeaseOnCloseException = dfsClient.getConfiguration().getRecoverLeaseOnCloseException(); try { if (isClosed()) { exceptionLastSeen.check(true); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index 445612f2f8397..dfc89adeab3bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -166,6 +166,7 @@ public class DfsClientConf { private final boolean deadNodeDetectionEnabled; private final long leaseHardLimitPeriod; + private final boolean recoverLeaseOnCloseException; public DfsClientConf(Configuration conf) { // The hdfsTimeout is currently the same as the ipc timeout @@ -315,6 +316,11 @@ public DfsClientConf(Configuration conf) { HdfsClientConfigKeys.DFS_CLIENT_PIPELINE_RECOVERY_MAX_RETRIES, HdfsClientConfigKeys.DFS_CLIENT_PIPELINE_RECOVERY_MAX_RETRIES_DEFAULT ); + + recoverLeaseOnCloseException = conf.getBoolean( + Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_KEY, + Write.RECOVER_LEASE_ON_CLOSE_EXCEPTION_DEFAULT + ); } private ByteArrayManager.Conf loadWriteByteArrayManagerConf( @@ -745,6 +751,10 @@ public int getMaxPipelineRecoveryRetries() { return maxPipelineRecoveryRetries; } + public boolean getRecoverLeaseOnCloseException() { + return recoverLeaseOnCloseException; + } + /** * Configuration for short-circuit reads. */ From 12e1f6ca96f48941224f98efd51c158934465f6c Mon Sep 17 00:00:00 2001 From: Abhey Rana Date: Tue, 4 Feb 2025 20:01:13 +0530 Subject: [PATCH 2/2] Fix build failures --- .../java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 7a59aa7012883..a6f703fcd43cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -1197,7 +1197,7 @@ void setClosed() { @Override protected synchronized void closeImpl() throws IOException { - boolean recoverLeaseOnCloseException = dfsClient.getConfiguration().getRecoverLeaseOnCloseException(); + boolean recoverLeaseOnCloseException = dfsClient.getConf().getRecoverLeaseOnCloseException(); try { if (isClosed()) { exceptionLastSeen.check(true);