From f3f6e9a59673e97b4085a8c60d24fc2981a9e1f2 Mon Sep 17 00:00:00 2001 From: "jinbiao.sun" Date: Tue, 5 Apr 2022 15:27:12 +0800 Subject: [PATCH 1/3] HDFS-16524. Add configuration to control blocks deletion asynchronous or synchronous --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++ .../hdfs/server/namenode/FSNamesystem.java | 52 ++++++++++++++----- .../src/main/resources/hdfs-default.xml | 8 +++ 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 6216f6e7a1ded..2595d89af8220 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -481,6 +481,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.block.deletion.increment"; public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000; + /** Block deletion asynchronous. */ + public static final String DFS_NAMENODE_BLOCK_DELETION_ASYNC_KEY = "dfs.namenode.block.deletion.async"; + public static final boolean DFS_NAMENODE_BLOCK_DELETION_ASYNC_DEFAULT = false; + public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES = HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES; public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 389bd6455c92f..930d65f96dccf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -502,6 +502,7 @@ private boolean isClientPortInfoAbsent(CallerContext ctx){ private final boolean standbyShouldCheckpoint; private final boolean isSnapshotTrashRootEnabled; private final int snapshotDiffReportLimit; + private final boolean blockDeletionAsync; private final int blockDeletionIncrement; /** @@ -1065,6 +1066,9 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { this.allowOwnerSetQuota = conf.getBoolean( DFSConfigKeys.DFS_PERMISSIONS_ALLOW_OWNER_SET_QUOTA_KEY, DFSConfigKeys.DFS_PERMISSIONS_ALLOW_OWNER_SET_QUOTA_DEFAULT); + this.blockDeletionAsync = conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_ASYNC_KEY, + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_ASYNC_DEFAULT); this.blockDeletionIncrement = conf.getInt( DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT); @@ -2387,8 +2391,7 @@ boolean truncate(String src, long newLength, String clientName, } getEditLog().logSync(); if (!toRemoveBlocks.getToDeleteList().isEmpty()) { - blockManager.addBLocksToMarkedDeleteQueue( - toRemoveBlocks.getToDeleteList()); + removeBlocks(toRemoveBlocks.getToDeleteList()); } logAuditEvent(true, operationName, src, null, status); } catch (AccessControlException e) { @@ -2835,8 +2838,7 @@ private HdfsFileStatus startFileInt(String src, if (!skipSync) { getEditLog().logSync(); if (toRemoveBlocks != null) { - blockManager.addBLocksToMarkedDeleteQueue( - toRemoveBlocks.getToDeleteList()); + removeBlocks(toRemoveBlocks.getToDeleteList()); } } } @@ -3359,8 +3361,7 @@ void renameTo(final String src, final String dst, assert res != null; BlocksMapUpdateInfo collectedBlocks = res.collectedBlocks; if (!collectedBlocks.getToDeleteList().isEmpty()) { - blockManager.addBLocksToMarkedDeleteQueue( - collectedBlocks.getToDeleteList()); + removeBlocks(collectedBlocks.getToDeleteList()); } logAuditEvent(true, operationName + " (options=" + @@ -3399,8 +3400,7 @@ boolean delete(String src, boolean recursive, boolean logRetryCache) getEditLog().logSync(); logAuditEvent(ret, operationName, src); if (toRemovedBlocks != null) { - blockManager.addBLocksToMarkedDeleteQueue( - toRemovedBlocks.getToDeleteList()); + removeBlocks(toRemovedBlocks.getToDeleteList()); } return ret; } @@ -3410,6 +3410,33 @@ FSPermissionChecker getPermissionChecker() return dir.getPermissionChecker(); } + /** + * If blockDeletionAsync enables, blocks will be deleted asynchronously. + * If not, incrementally remove the blocks from blockManager + * Writelock is dropped and reacquired every BLOCK_DELETION_INCREMENT to + * ensure that other waiters on the lock can get in. See HDFS-2938 + * + * @param toDeleteList + * a list of blocks that need to be removed from blocksMap + */ + void removeBlocks(List toDeleteList) { + if (this.blockDeletionAsync) { + blockManager.addBLocksToMarkedDeleteQueue(toDeleteList); + } else { + Iterator iter = toDeleteList.iterator(); + while (iter.hasNext()) { + writeLock(); + try { + for (int i = 0; i < blockDeletionIncrement && iter.hasNext(); i++) { + blockManager.removeBlock(iter.next()); + } + } finally { + writeUnlock("removeBlocks"); + } + } + } + } + /** * Remove leases and inodes related to a given path * @param removedUCFiles INodes whose leases need to be released @@ -4618,8 +4645,7 @@ private void clearCorruptLazyPersistFiles() INodesInPath.fromINode((INodeFile) bc), false); changed |= toRemoveBlocks != null; if (toRemoveBlocks != null) { - blockManager.addBLocksToMarkedDeleteQueue( - toRemoveBlocks.getToDeleteList()); + removeBlocks(toRemoveBlocks.getToDeleteList()); } } } finally { @@ -7330,8 +7356,7 @@ void deleteSnapshot(String snapshotRoot, String snapshotName, // Breaking the pattern as removing blocks have to happen outside of the // global lock if (blocksToBeDeleted != null) { - blockManager.addBLocksToMarkedDeleteQueue( - blocksToBeDeleted.getToDeleteList()); + removeBlocks(blocksToBeDeleted.getToDeleteList()); } logAuditEvent(true, operationName, rootPath, null, null); } @@ -7357,8 +7382,7 @@ public void gcDeletedSnapshot(String snapshotRoot, String snapshotName) } finally { writeUnlock(operationName, getLockReportInfoSupplier(rootPath)); } - blockManager.addBLocksToMarkedDeleteQueue( - blocksToBeDeleted.getToDeleteList()); + removeBlocks(blocksToBeDeleted.getToDeleteList()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index d45f8eb5b7ec5..cfd3b809c91b1 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -6128,6 +6128,14 @@ + + dfs.namenode.block.deletion.async + false + + If true, enable block deleting asynchronously + + + dfs.namenode.rpc-address.auxiliary-ports From 3c835c1b1edb72bac09f15a6a81a7cb8677232b1 Mon Sep 17 00:00:00 2001 From: "jinbiao.sun" Date: Sun, 31 Jul 2022 22:24:37 +0800 Subject: [PATCH 2/3] HDFS-16524. Set dfs.namenode.block.deletion.async default value to true --- .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 2595d89af8220..6ef33d62be5b8 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -483,7 +483,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { /** Block deletion asynchronous. */ public static final String DFS_NAMENODE_BLOCK_DELETION_ASYNC_KEY = "dfs.namenode.block.deletion.async"; - public static final boolean DFS_NAMENODE_BLOCK_DELETION_ASYNC_DEFAULT = false; + public static final boolean DFS_NAMENODE_BLOCK_DELETION_ASYNC_DEFAULT = true; public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES = HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES; From 8f55bb8a1d5c7e1405dd74a15b1e387a7d8688ab Mon Sep 17 00:00:00 2001 From: "jinbiao.sun" Date: Sat, 3 Sep 2022 15:41:58 +0800 Subject: [PATCH 3/3] HDFS-16524. Recover the original config dfs.namenode.block.deletion.increment --- .../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++ .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 8 ++++++++ .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 4 ++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 4a24590a25c00..8e405ac09b352 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -495,6 +495,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS_DEFAULT = 10; + /** Block deletion increment. */ + public static final String DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY = + "dfs.namenode.block.deletion.increment"; + public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000; /** Block deletion asynchronous. */ public static final String DFS_NAMENODE_BLOCK_DELETION_ASYNC_KEY = "dfs.namenode.block.deletion.async"; public static final boolean DFS_NAMENODE_BLOCK_DELETION_ASYNC_DEFAULT = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index f466bf0aee402..95a8cca87d7b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -501,6 +501,8 @@ private boolean isClientPortInfoAbsent(CallerContext ctx){ private final String supergroup; private final boolean standbyShouldCheckpoint; private final boolean isSnapshotTrashRootEnabled; + private final boolean blockDeletionAsync; + private final int blockDeletionIncrement; private final int snapshotDiffReportLimit; /** @@ -1064,6 +1066,12 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { this.allowOwnerSetQuota = conf.getBoolean( DFSConfigKeys.DFS_PERMISSIONS_ALLOW_OWNER_SET_QUOTA_KEY, DFSConfigKeys.DFS_PERMISSIONS_ALLOW_OWNER_SET_QUOTA_DEFAULT); + this.blockDeletionAsync = conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_ASYNC_KEY, + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_ASYNC_DEFAULT); + this.blockDeletionIncrement = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT); this.isGetBlocksCheckOperationEnabled = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY, DFSConfigKeys.DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 7b25f28a3f504..15a64db81973b 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -6165,9 +6165,9 @@ dfs.namenode.block.deletion.async - false + true - If true, enable block deleting asynchronously + If false, disable block deleting asynchronously