From 983f54df703b02806d47628ff2ddbc8392831e0e Mon Sep 17 00:00:00 2001 From: zhanghaobo Date: Thu, 6 Mar 2025 21:10:44 +0800 Subject: [PATCH 1/4] HDFS-17705. Improve datasetlock related log output. --- .../hdfs/server/datanode/DataSetLockManager.java | 10 ++++++++-- .../server/datanode/fsdataset/impl/FsDatasetImpl.java | 6 ++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java index dafbb4ed50227..dc8c1e87fe18c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java @@ -179,7 +179,7 @@ public AutoCloseDataSetLock readLock(LockLevel level, String... resources) { AutoCloseDataSetLock dirLock = getReadLock(level, resources); dirLock.setParentLock(volLock); if (openLockTrace) { - LOG.debug("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " + + LOG.info("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " + resources[0] + resources[1]); } return dirLock; @@ -206,7 +206,7 @@ public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) { AutoCloseDataSetLock dirLock = getWriteLock(level, resources); dirLock.setParentLock(volLock); if (openLockTrace) { - LOG.debug("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " + + LOG.info("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " + resources[0] + resources[1]); } return dirLock; @@ -273,6 +273,9 @@ public void addLock(LockLevel level, String... resources) { new ReentrantReadWriteLock(isFair)); lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair)); } + if (openLockTrace) { + LOG.info("Added {} lock, lock name: {}", level.name(), lockName); + } } @Override @@ -281,6 +284,9 @@ public void removeLock(LockLevel level, String... resources) { try (AutoCloseDataSetLock lock = writeLock(level, resources)) { lockMap.removeLock(lockName); } + if (openLockTrace) { + LOG.info("Removed {} lock, lock name: {}", level.name(), lockName); + } } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 934c5faee2f92..e6c353279dafb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -439,9 +439,8 @@ private synchronized void activateVolume( List allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockNames(); for (String dir : allSubDirNameForDataSetLock) { lockManager.addLock(LockLevel.DIR, bp, ref.getVolume().getStorageID(), dir); - LOG.info("Added DIR lock for bpid:{}, volume storageid:{}, dir:{}", - bp, ref.getVolume().getStorageID(), dir); } + LOG.info("Added DIR lock for bpid:{}, volume storageid:{}.", bp, ref.getVolume().getStorageID()); } DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid()); if (dnStorage != null) { @@ -3297,9 +3296,8 @@ public void addBlockPool(String bpid, Configuration conf) List allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockNames(); for (String dir : allSubDirNameForDataSetLock) { lockManager.addLock(LockLevel.DIR, bpid, v, dir); - LOG.info("Added DIR lock for bpid:{}, volume storageid:{}, dir:{}", - bpid, v, dir); } + LOG.info("Added DIR lock for bpid:{}, volume storageid:{}.", bpid, v); } } try { From 65e3347d2ca2c1eb0a3a924ae07de0146b006fbe Mon Sep 17 00:00:00 2001 From: zhanghaobo Date: Thu, 6 Mar 2025 23:52:00 +0800 Subject: [PATCH 2/4] fix checkstyle. --- .../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index e6c353279dafb..0e5aab062074f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -440,7 +440,8 @@ private synchronized void activateVolume( for (String dir : allSubDirNameForDataSetLock) { lockManager.addLock(LockLevel.DIR, bp, ref.getVolume().getStorageID(), dir); } - LOG.info("Added DIR lock for bpid:{}, volume storageid:{}.", bp, ref.getVolume().getStorageID()); + LOG.info("Added DIR lock for bpid:{}, volume storageid:{}.", bp, + ref.getVolume().getStorageID()); } DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid()); if (dnStorage != null) { From 72e84883bd4a1282253065506a148f332ddf3c81 Mon Sep 17 00:00:00 2001 From: zhanghaobo Date: Wed, 12 Mar 2025 11:26:33 +0800 Subject: [PATCH 3/4] trigger yetus. From 8ef0228e6dbf1969aa31e3b44dc451cc3ada5f1e Mon Sep 17 00:00:00 2001 From: zhanghaobo Date: Thu, 13 Mar 2025 09:38:21 +0800 Subject: [PATCH 4/4] fix log format --- .../hdfs/server/datanode/DataSetLockManager.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java index dc8c1e87fe18c..b127ebf6f8351 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java @@ -168,8 +168,7 @@ public AutoCloseDataSetLock readLock(LockLevel level, String... resources) { AutoCloseDataSetLock volLock = getReadLock(level, resources); volLock.setParentLock(bpLock); if (openLockTrace) { - LOG.info("Sub lock " + resources[0] + resources[1] + " parent lock " + - resources[0]); + LOG.info("Sub lock {}, parent lock {}.", resources[0] + resources[1], resources[0]); } return volLock; } else { @@ -179,7 +178,7 @@ public AutoCloseDataSetLock readLock(LockLevel level, String... resources) { AutoCloseDataSetLock dirLock = getReadLock(level, resources); dirLock.setParentLock(volLock); if (openLockTrace) { - LOG.info("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " + + LOG.info("Sub lock {}, parent lock {}", resources[0] + resources[1] + resources[2], resources[0] + resources[1]); } return dirLock; @@ -195,8 +194,7 @@ public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) { AutoCloseDataSetLock volLock = getWriteLock(level, resources); volLock.setParentLock(bpLock); if (openLockTrace) { - LOG.info("Sub lock " + resources[0] + resources[1] + " parent lock " + - resources[0]); + LOG.info("Sub lock {}, parent lock {}.", resources[0] + resources[1], resources[0]); } return volLock; } else { @@ -206,7 +204,7 @@ public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) { AutoCloseDataSetLock dirLock = getWriteLock(level, resources); dirLock.setParentLock(volLock); if (openLockTrace) { - LOG.info("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " + + LOG.info("Sub lock {}, parent lock {}.", resources[0] + resources[1] + resources[2], resources[0] + resources[1]); } return dirLock;