From 374a2bdc9fc5a2c5ee6582585223dd49c0f18cfe Mon Sep 17 00:00:00 2001
From: "zhanghaobo@kanzhun.com" <hfutzhanghb@163.com>
Date: Tue, 23 Apr 2024 16:39:18 +0800
Subject: [PATCH 01/22] HDFS-17496. DataNode supports more fine-grained dataset
 lock based on blockid.

---
 .../server/common/DataNodeLockManager.java    |   3 +-
 .../server/datanode/DataSetLockManager.java   |  33 ++++-
 .../hdfs/server/datanode/DatanodeUtil.java    |  26 ++++
 .../fsdataset/impl/FsDatasetImpl.java         | 118 ++++++++++++------
 .../datanode/TestDataSetLockManager.java      |  11 ++
 5 files changed, 147 insertions(+), 44 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java
index e7a3b38357ac9..cb22a0570627f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java
@@ -29,7 +29,8 @@ public interface DataNodeLockManager<T extends AutoCloseDataSetLock> {
    */
   enum LockLevel {
     BLOCK_POOl,
-    VOLUME
+    VOLUME,
+    DIR
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
index 3abcf12fc8b12..28fdac54e6f74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
@@ -96,6 +96,8 @@ private String generateLockName(LockLevel level, String... resources) {
             + resources[0] + "volume lock :" + resources[1]);
       }
       return resources[0] + resources[1];
+    } else if (resources.length == 3 && level == LockLevel.DIR) {
+      return resources[0] + resources[1] + resources[2];
     } else {
       throw new IllegalArgumentException("lock level do not match resource");
     }
@@ -156,7 +158,7 @@ public DataSetLockManager(Configuration conf, DataNode dn) {
   public AutoCloseDataSetLock readLock(LockLevel level, String... resources) {
     if (level == LockLevel.BLOCK_POOl) {
       return getReadLock(level, resources[0]);
-    } else {
+    } else if (level == LockLevel.VOLUME){
       AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]);
       AutoCloseDataSetLock volLock = getReadLock(level, resources);
       volLock.setParentLock(bpLock);
@@ -165,6 +167,17 @@ public AutoCloseDataSetLock readLock(LockLevel level, String... resources) {
             resources[0]);
       }
       return volLock;
+    } else {
+      AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]);
+      AutoCloseDataSetLock volLock = getReadLock(LockLevel.VOLUME, resources[0], resources[1]);
+      volLock.setParentLock(bpLock);
+      AutoCloseDataSetLock dirLock = getWriteLock(level, resources);
+      dirLock.setParentLock(volLock);
+      if (openLockTrace) {
+        LOG.debug("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " +
+            resources[0] + resources[1]);
+      }
+      return dirLock;
     }
   }
 
@@ -172,7 +185,7 @@ public AutoCloseDataSetLock readLock(LockLevel level, String... resources) {
   public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) {
     if (level == LockLevel.BLOCK_POOl) {
       return getWriteLock(level, resources[0]);
-    } else {
+    } else if (level == LockLevel.VOLUME) {
       AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]);
       AutoCloseDataSetLock volLock = getWriteLock(level, resources);
       volLock.setParentLock(bpLock);
@@ -181,6 +194,17 @@ public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) {
             resources[0]);
       }
       return volLock;
+    } else {
+      AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]);
+      AutoCloseDataSetLock volLock = getReadLock(LockLevel.VOLUME, resources[0], resources[1]);
+      volLock.setParentLock(bpLock);
+      AutoCloseDataSetLock dirLock = getReadLock(level, resources);
+      dirLock.setParentLock(volLock);
+      if (openLockTrace) {
+        LOG.debug("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " +
+            resources[0] + resources[1]);
+      }
+      return dirLock;
     }
   }
 
@@ -235,8 +259,13 @@ public void addLock(LockLevel level, String... resources) {
     String lockName = generateLockName(level, resources);
     if (level == LockLevel.BLOCK_POOl) {
       lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair));
+    } else if (level == LockLevel.VOLUME) {
+      lockMap.addLock(resources[0], new ReentrantReadWriteLock(isFair));
+      lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair));
     } else {
       lockMap.addLock(resources[0], new ReentrantReadWriteLock(isFair));
+      lockMap.addLock(generateLockName(LockLevel.VOLUME, resources[0], resources[1]),
+          new ReentrantReadWriteLock(isFair));
       lockMap.addLock(lockName, new ReentrantReadWriteLock(isFair));
     }
   }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index c98ff5413bd85..af7603178eae1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -21,6 +21,8 @@
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -127,6 +129,30 @@ public static File idToBlockDir(File root, long blockId) {
     return new File(root, path);
   }
 
+  /**
+   * Take an example: we hava a block with blockid mapping to:
+   * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir0"
+   * We return "subdir0/subdir0"
+   * @return
+   */
+  public static String idToBlockDirSuffixName(long blockId) {
+    int d1 = (int) ((blockId >> 16) & 0x1F);
+    int d2 = (int) ((blockId >> 8) & 0x1F);
+    return DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
+        DataStorage.BLOCK_SUBDIR_PREFIX + d2;
+  }
+  
+  public static List<String> getAllSubDirNameForDataSetLock() {
+    List<String> res = new ArrayList<>();
+    for (int d1 = 0; d1 <= 0x1F; d1++) {
+      for (int d2 = 0; d2 <= 0x1F; d2++) {
+        res.add(DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
+            DataStorage.BLOCK_SUBDIR_PREFIX + d2);
+      }
+    }
+    return res;
+  }
+
   /**
    * @return the FileInputStream for the meta data of the given block.
    * @throws FileNotFoundException
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index eeec1bb728825..8347a332b65a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -138,6 +138,7 @@
 class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
   static final Logger LOG = LoggerFactory.getLogger(FsDatasetImpl.class);
   private final static boolean isNativeIOAvailable;
+  private static final String SEP = System.getProperty("file.separator");
   private Timer timer;
   static {
     isNativeIOAvailable = NativeIO.isAvailable();
@@ -198,8 +199,9 @@ public FsVolumeImpl getVolume(final ExtendedBlock b) {
   @Override // FsDatasetSpi
   public Block getStoredBlock(String bpid, long blkid)
       throws IOException {
-    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
-        bpid)) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.DIR,
+        bpid, getReplicaInfo(bpid, blkid).getStorageUuid(),
+        DatanodeUtil.idToBlockDirSuffixName(blkid))) {
       ReplicaInfo r = volumeMap.get(bpid, blkid);
       if (r == null) {
         return null;
@@ -430,6 +432,13 @@ private synchronized void activateVolume(
       FsVolumeReference ref) throws IOException {
     for (String bp : volumeMap.getBlockPoolList()) {
       lockManager.addLock(LockLevel.VOLUME, bp, ref.getVolume().getStorageID());
+      List<String> allSubDirNameForDataSetLock = DatanodeUtil.getAllSubDirNameForDataSetLock();
+      for (String dir : allSubDirNameForDataSetLock) {
+        lockManager.addLock(LockLevel.DIR, bp, ref.getVolume().getStorageID(), dir);
+        LOG.info("Added DIR lock for bpid:{}, volume storageid:{}, dir:{}",
+            bp, ref.getVolume().getStorageID(), dir);
+        
+      }
     }
     DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid());
     if (dnStorage != null) {
@@ -629,6 +638,12 @@ public void removeVolumes(
       for (String storageUuid : storageToRemove) {
         storageMap.remove(storageUuid);
         for (String bp : volumeMap.getBlockPoolList()) {
+          List<String> allSubDirNameForDataSetLock = DatanodeUtil.getAllSubDirNameForDataSetLock();
+          for (String dir : allSubDirNameForDataSetLock) {
+            lockManager.removeLock(LockLevel.DIR, bp, storageUuid, dir);
+            LOG.info("Removed DIR lock for bpid:{}, volume storageid:{}, dir:{}",
+                bp, storageUuid, dir);
+          }
           lockManager.removeLock(LockLevel.VOLUME, bp, storageUuid);
         }
       }
@@ -819,8 +834,9 @@ public InputStream getBlockInputStream(ExtendedBlock b,
       long seekOffset) throws IOException {
 
     ReplicaInfo info;
-    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
-        b.getBlockPoolId())) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.DIR,
+        b.getBlockPoolId(), getStorageUuidForLock(b),
+        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
       info = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock());
     }
 
@@ -914,8 +930,9 @@ String getStorageUuidForLock(ExtendedBlock b)
   @Override // FsDatasetSpi
   public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b,
       long blkOffset, long metaOffset) throws IOException {
-    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.VOLUME,
-        b.getBlockPoolId(), getStorageUuidForLock(b))) {
+    try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.DIR,
+        b.getBlockPoolId(), getStorageUuidForLock(b),
+        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
       ReplicaInfo info = getReplicaInfo(b);
       FsVolumeReference ref = info.getVolume().obtainReference();
       try {
@@ -1380,8 +1397,9 @@ static void computeChecksum(ReplicaInfo srcReplica, File dstMeta,
   @Override  // FsDatasetSpi
   public ReplicaHandler append(ExtendedBlock b,
       long newGS, long expectedBlockLen) throws IOException {
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-        b.getBlockPoolId(), getStorageUuidForLock(b))) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+        b.getBlockPoolId(), getStorageUuidForLock(b),
+        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
       // If the block was successfully finalized because all packets
       // were successfully processed at the Datanode but the ack for
       // some of the packets were not received by the client. The client
@@ -1433,8 +1451,9 @@ public ReplicaHandler append(ExtendedBlock b,
   private ReplicaInPipeline append(String bpid,
       ReplicaInfo replicaInfo, long newGS, long estimateBlockLen)
       throws IOException {
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-        bpid, replicaInfo.getStorageUuid())) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+        bpid, replicaInfo.getStorageUuid(),
+        DatanodeUtil.idToBlockDirSuffixName(replicaInfo.getBlockId()))) {
       // If the block is cached, start uncaching it.
       if (replicaInfo.getState() != ReplicaState.FINALIZED) {
         throw new IOException("Only a Finalized replica can be appended to; "
@@ -1530,8 +1549,8 @@ public ReplicaHandler recoverAppend(
 
     while (true) {
       try {
-        try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl,
-            b.getBlockPoolId())) {
+        try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+            b.getBlockPoolId(), getStorageUuidForLock(b), DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
           ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
           FsVolumeReference ref = replicaInfo.getVolume().obtainReference();
           ReplicaInPipeline replica;
@@ -1564,8 +1583,9 @@ public Replica recoverClose(ExtendedBlock b, long newGS,
         b, newGS, expectedBlockLen);
     while (true) {
       try {
-        try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-            b.getBlockPoolId(), getStorageUuidForLock(b))) {
+        try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+            b.getBlockPoolId(), getStorageUuidForLock(b),
+            DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
           // check replica's state
           ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
           // bump the replica's GS
@@ -1650,8 +1670,9 @@ public ReplicaHandler createRbw(
       }
 
       ReplicaInPipeline newReplicaInfo;
-      try (AutoCloseableLock l = lockManager.writeLock(LockLevel.VOLUME,
-          b.getBlockPoolId(), v.getStorageID())) {
+      try (AutoCloseableLock l = lockManager.writeLock(LockLevel.DIR,
+          b.getBlockPoolId(), v.getStorageID(),
+          DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
         newReplicaInfo = v.createRbw(b);
         if (newReplicaInfo.getReplicaInfo().getState() != ReplicaState.RBW) {
           throw new IOException("CreateRBW returned a replica of state "
@@ -1681,8 +1702,9 @@ public ReplicaHandler recoverRbw(
     try {
       while (true) {
         try {
-          try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-              b.getBlockPoolId(), getStorageUuidForLock(b))) {
+          try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+              b.getBlockPoolId(), getStorageUuidForLock(b),
+              DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
             ReplicaInfo replicaInfo =
                 getReplicaInfo(b.getBlockPoolId(), b.getBlockId());
             // check the replica's state
@@ -1713,8 +1735,9 @@ public ReplicaHandler recoverRbw(
   private ReplicaHandler recoverRbwImpl(ReplicaInPipeline rbw,
       ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd)
       throws IOException {
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-        b.getBlockPoolId(), getStorageUuidForLock(b))) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+        b.getBlockPoolId(), getStorageUuidForLock(b),
+        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
       // check generation stamp
       long replicaGenerationStamp = rbw.getGenerationStamp();
       if (replicaGenerationStamp < b.getGenerationStamp() ||
@@ -1775,8 +1798,9 @@ private ReplicaHandler recoverRbwImpl(ReplicaInPipeline rbw,
   public ReplicaInPipeline convertTemporaryToRbw(
       final ExtendedBlock b) throws IOException {
     long startTimeMs = Time.monotonicNow();
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-        b.getBlockPoolId(), getStorageUuidForLock(b))) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+        b.getBlockPoolId(), getStorageUuidForLock(b),
+        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
       final long blockId = b.getBlockId();
       final long expectedGs = b.getGenerationStamp();
       final long visible = b.getNumBytes();
@@ -1915,8 +1939,8 @@ public ReplicaHandler createTemporary(StorageType storageType,
         .getNumBytes());
     FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
     ReplicaInPipeline newReplicaInfo;
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-        b.getBlockPoolId(), v.getStorageID())) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+        b.getBlockPoolId(), v.getStorageID(), DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
       try {
         newReplicaInfo = v.createTemporary(b);
         LOG.debug("creating temporary for block: {} on volume: {}",
@@ -1973,8 +1997,9 @@ public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
     ReplicaInfo replicaInfo = null;
     ReplicaInfo finalizedReplicaInfo = null;
     long startTimeMs = Time.monotonicNow();
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-        b.getBlockPoolId(), getStorageUuidForLock(b))) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+        b.getBlockPoolId(), getStorageUuidForLock(b),
+        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
       if (Thread.interrupted()) {
         // Don't allow data modifications from interrupted threads
         throw new IOException("Cannot finalize block: " + b + " from Interrupted Thread");
@@ -2010,8 +2035,9 @@ public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
 
   private ReplicaInfo finalizeReplica(String bpid, ReplicaInfo replicaInfo)
       throws IOException {
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-        bpid, replicaInfo.getStorageUuid())) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+        bpid, replicaInfo.getStorageUuid(),
+        DatanodeUtil.idToBlockDirSuffixName(replicaInfo.getBlockId()))) {
       // Compare generation stamp of old and new replica before finalizing
       if (volumeMap.get(bpid, replicaInfo.getBlockId()).getGenerationStamp()
           > replicaInfo.getGenerationStamp()) {
@@ -2060,8 +2086,9 @@ private ReplicaInfo finalizeReplica(String bpid, ReplicaInfo replicaInfo)
   @Override // FsDatasetSpi
   public void unfinalizeBlock(ExtendedBlock b) throws IOException {
     long startTimeMs = Time.monotonicNow();
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,
-        b.getBlockPoolId(), getStorageUuidForLock(b))) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
+        b.getBlockPoolId(), getStorageUuidForLock(b),
+        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
       ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
           b.getLocalBlock());
       if (replicaInfo != null &&
@@ -2459,7 +2486,8 @@ boolean removeReplicaFromMem(final ExtendedBlock block, final FsVolumeImpl volum
     final String bpid = block.getBlockPoolId();
     final Block localBlock = block.getLocalBlock();
     final long blockId = localBlock.getBlockId();
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR, bpid, volume.getStorageID(),
+        DatanodeUtil.idToBlockDirSuffixName(blockId))) {
       final ReplicaInfo info = volumeMap.get(bpid, localBlock);
       if (info == null) {
         ReplicaInfo infoByBlockId = volumeMap.get(bpid, blockId);
@@ -2548,8 +2576,8 @@ private void cacheBlock(String bpid, long blockId) {
           bpid + ": ReplicaInfo not found.");
       return;
     }
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, bpid,
-        info.getStorageUuid())) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR, bpid,
+        info.getStorageUuid(), DatanodeUtil.idToBlockDirSuffixName(blockId))) {
       boolean success = false;
       try {
         info = volumeMap.get(bpid, blockId);
@@ -3002,8 +3030,8 @@ ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map,
         }
         LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId
             + ", replica=" + replica);
-        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.VOLUME, bpid,
-            replica.getStorageUuid())) {
+        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.DIR, bpid,
+            replica.getStorageUuid(), DatanodeUtil.idToBlockDirSuffixName(block.getBlockId()))) {
           return initReplicaRecoveryImpl(bpid, map, block, recoveryId);
         }
       } catch (MustStopExistingWriter e) {
@@ -3024,8 +3052,8 @@ static ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map,
         }
         LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId
             + ", replica=" + replica);
-        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.VOLUME, bpid,
-            replica.getStorageUuid())) {
+        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.DIR, bpid,
+            replica.getStorageUuid(), DatanodeUtil.idToBlockDirSuffixName(block.getBlockId()))) {
           return initReplicaRecoveryImpl(bpid, map, block, recoveryId);
         }
       } catch (MustStopExistingWriter e) {
@@ -3231,8 +3259,9 @@ private ReplicaInfo updateReplicaUnderRecovery(
   @Override // FsDatasetSpi
   public long getReplicaVisibleLength(final ExtendedBlock block)
   throws IOException {
-    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
-        block.getBlockPoolId())) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.DIR,
+        block.getBlockPoolId(), getStorageUuidForLock(block),
+        DatanodeUtil.idToBlockDirSuffixName(block.getBlockId()))) {
       final Replica replica = getReplicaInfo(block.getBlockPoolId(),
           block.getBlockId());
       if (replica.getGenerationStamp() < block.getGenerationStamp()) {
@@ -3259,6 +3288,12 @@ public void addBlockPool(String bpid, Configuration conf)
       Set<String> vols = storageMap.keySet();
       for (String v : vols) {
         lockManager.addLock(LockLevel.VOLUME, bpid, v);
+        List<String> allSubDirNameForDataSetLock = DatanodeUtil.getAllSubDirNameForDataSetLock();
+        for (String dir : allSubDirNameForDataSetLock) {
+          lockManager.addLock(LockLevel.DIR, bpid, v, dir);
+          LOG.info("Added DIR lock for bpid:{}, volume storageid:{}, dir:{}",
+              bpid, v, dir);
+        }
       }
     }
     try {
@@ -3386,8 +3421,9 @@ public void deleteBlockPool(String bpid, boolean force)
   @Override // FsDatasetSpi
   public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block)
       throws IOException {
-    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.BLOCK_POOl,
-        block.getBlockPoolId())) {
+    try (AutoCloseableLock lock = lockManager.readLock(LockLevel.DIR,
+        block.getBlockPoolId(), getStorageUuidForLock(block),
+        DatanodeUtil.idToBlockDirSuffixName(block.getBlockId()))) {
       final Replica replica = volumeMap.get(block.getBlockPoolId(),
           block.getBlockId());
       if (replica == null) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
index b514accdf16e9..9efdc3f89a684 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
@@ -37,6 +37,7 @@ public void init() {
   public void testBaseFunc() {
     manager.addLock(LockLevel.BLOCK_POOl, "BPtest");
     manager.addLock(LockLevel.VOLUME, "BPtest", "Volumetest");
+    manager.addLock(LockLevel.DIR, "BPtest", "Volumetest", "SubDirtest");
 
     AutoCloseDataSetLock lock = manager.writeLock(LockLevel.BLOCK_POOl, "BPtest");
     AutoCloseDataSetLock lock1 = manager.readLock(LockLevel.BLOCK_POOl, "BPtest");
@@ -62,6 +63,15 @@ public void testBaseFunc() {
     manager.lockLeakCheck();
     assertNull(manager.getLastException());
 
+    AutoCloseDataSetLock lock6 = manager.writeLock(LockLevel.BLOCK_POOl, "BPtest");
+    AutoCloseDataSetLock lock7 = manager.readLock(LockLevel.VOLUME, "BPtest", "Volumetest");
+    AutoCloseDataSetLock lock8 = manager.readLock(LockLevel.DIR, "BPtest", "Volumetest", "SubDirtest");
+    lock8.close();
+    lock7.close();
+    lock6.close();
+    manager.lockLeakCheck();
+    assertNull(manager.getLastException());
+
     manager.writeLock(LockLevel.VOLUME, "BPtest", "Volumetest");
     manager.lockLeakCheck();
 
@@ -89,4 +99,5 @@ public void testLockLeakCheck() {
     Exception lastException = manager.getLastException();
     assertEquals(lastException.getMessage(), "lock Leak");
   }
+
 }

From a044c83e5b015891c8db64a1244c87901b37d5b3 Mon Sep 17 00:00:00 2001
From: "zhanghaobo@kanzhun.com" <hfutzhanghb@163.com>
Date: Tue, 23 Apr 2024 16:45:22 +0800
Subject: [PATCH 02/22] remove used statement.

---
 .../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java       | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 8347a332b65a9..cdd2f202ccbd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -138,7 +138,6 @@
 class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
   static final Logger LOG = LoggerFactory.getLogger(FsDatasetImpl.class);
   private final static boolean isNativeIOAvailable;
-  private static final String SEP = System.getProperty("file.separator");
   private Timer timer;
   static {
     isNativeIOAvailable = NativeIO.isAvailable();

From a78e0f123302dea2cdcce932efa78a5a8e1bdbfc Mon Sep 17 00:00:00 2001
From: "zhanghaobo@kanzhun.com" <hfutzhanghb@163.com>
Date: Fri, 28 Jun 2024 11:04:49 +0800
Subject: [PATCH 03/22] fix bug

---
 .../hadoop/hdfs/server/datanode/DataSetLockManager.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
index 28fdac54e6f74..1e58d6b3500a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
@@ -171,7 +171,7 @@ public AutoCloseDataSetLock readLock(LockLevel level, String... resources) {
       AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]);
       AutoCloseDataSetLock volLock = getReadLock(LockLevel.VOLUME, resources[0], resources[1]);
       volLock.setParentLock(bpLock);
-      AutoCloseDataSetLock dirLock = getWriteLock(level, resources);
+      AutoCloseDataSetLock dirLock = getReadLock(level, resources);
       dirLock.setParentLock(volLock);
       if (openLockTrace) {
         LOG.debug("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " +
@@ -198,7 +198,7 @@ public AutoCloseDataSetLock writeLock(LockLevel level, String... resources) {
       AutoCloseDataSetLock bpLock = getReadLock(LockLevel.BLOCK_POOl, resources[0]);
       AutoCloseDataSetLock volLock = getReadLock(LockLevel.VOLUME, resources[0], resources[1]);
       volLock.setParentLock(bpLock);
-      AutoCloseDataSetLock dirLock = getReadLock(level, resources);
+      AutoCloseDataSetLock dirLock = getWriteLock(level, resources);
       dirLock.setParentLock(volLock);
       if (openLockTrace) {
         LOG.debug("Sub lock " + resources[0] + resources[1] + resources[2] + " parent lock " +

From 0ac3b07fca9e773e83a991b96e71560029fa2d14 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Wed, 27 Nov 2024 14:26:58 +0800
Subject: [PATCH 04/22] fix checkstyles.

---
 .../apache/hadoop/hdfs/server/datanode/DatanodeUtil.java   | 4 ++--
 .../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java | 7 ++++---
 .../hdfs/server/datanode/TestDataSetLockManager.java       | 3 ++-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index af7603178eae1..3b3e9444f59df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -130,7 +130,7 @@ public static File idToBlockDir(File root, long blockId) {
   }
 
   /**
-   * Take an example: we hava a block with blockid mapping to:
+   * Take an example. We hava a block with blockid mapping to:
    * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir0"
    * We return "subdir0/subdir0"
    * @return
@@ -141,7 +141,7 @@ public static String idToBlockDirSuffixName(long blockId) {
     return DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
         DataStorage.BLOCK_SUBDIR_PREFIX + d2;
   }
-  
+
   public static List<String> getAllSubDirNameForDataSetLock() {
     List<String> res = new ArrayList<>();
     for (int d1 = 0; d1 <= 0x1F; d1++) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index cdd2f202ccbd6..c118190827ba6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -436,7 +436,6 @@ private synchronized void activateVolume(
         lockManager.addLock(LockLevel.DIR, bp, ref.getVolume().getStorageID(), dir);
         LOG.info("Added DIR lock for bpid:{}, volume storageid:{}, dir:{}",
             bp, ref.getVolume().getStorageID(), dir);
-        
       }
     }
     DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid());
@@ -1549,7 +1548,8 @@ public ReplicaHandler recoverAppend(
     while (true) {
       try {
         try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
-            b.getBlockPoolId(), getStorageUuidForLock(b), DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+            b.getBlockPoolId(), getStorageUuidForLock(b),
+            DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
           ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
           FsVolumeReference ref = replicaInfo.getVolume().obtainReference();
           ReplicaInPipeline replica;
@@ -1939,7 +1939,8 @@ public ReplicaHandler createTemporary(StorageType storageType,
     FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
     ReplicaInPipeline newReplicaInfo;
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
-        b.getBlockPoolId(), v.getStorageID(), DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+        b.getBlockPoolId(), v.getStorageID(),
+        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
       try {
         newReplicaInfo = v.createTemporary(b);
         LOG.debug("creating temporary for block: {} on volume: {}",
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
index 9efdc3f89a684..533aae6e27109 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
@@ -65,7 +65,8 @@ public void testBaseFunc() {
 
     AutoCloseDataSetLock lock6 = manager.writeLock(LockLevel.BLOCK_POOl, "BPtest");
     AutoCloseDataSetLock lock7 = manager.readLock(LockLevel.VOLUME, "BPtest", "Volumetest");
-    AutoCloseDataSetLock lock8 = manager.readLock(LockLevel.DIR, "BPtest", "Volumetest", "SubDirtest");
+    AutoCloseDataSetLock lock8 = manager.readLock(LockLevel.DIR,
+        "BPtest", "Volumetest", "SubDirtest");
     lock8.close();
     lock7.close();
     lock6.close();

From 30005539f9eb136e1241c1458e18ad3566a7f5da Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Wed, 27 Nov 2024 14:32:06 +0800
Subject: [PATCH 05/22] fix checkstyles.

---
 .../hadoop/hdfs/server/datanode/TestDataSetLockManager.java      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
index 533aae6e27109..6cb12d2681f82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataSetLockManager.java
@@ -100,5 +100,4 @@ public void testLockLeakCheck() {
     Exception lastException = manager.getLastException();
     assertEquals(lastException.getMessage(), "lock Leak");
   }
-
 }

From 9308f211c05877a214c7a5f521940e43603199df Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Thu, 28 Nov 2024 23:43:19 +0800
Subject: [PATCH 06/22] fix failed UTs

---
 .../datanode/fsdataset/impl/TestFsDatasetImpl.java    |  8 +++++++-
 .../hadoop/hdfs/server/namenode/ha/TestDNFencing.java | 11 ++++++++---
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index 975874edb1fc0..fa677f0ceab0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -68,6 +68,7 @@
 import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaHandler;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException;
 import org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
@@ -1946,7 +1947,12 @@ public void delayDeleteReplica() {
       assertFalse(uuids.contains(dn.getDatanodeUuid()));
 
       // This replica has deleted from datanode memory.
-      assertNull(ds.getStoredBlock(bpid, extendedBlock.getBlockId()));
+      try {
+        Block storedBlock = ds.getStoredBlock(bpid, extendedBlock.getBlockId());
+        assertNull(storedBlock);
+      } catch (Exception e) {
+        GenericTestUtils.assertExceptionContains("ReplicaNotFoundException", e);
+      }
     } finally {
       cluster.shutdown();
       DataNodeFaultInjector.set(oldInjector);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
index 177b7bca4ce82..fa84a204a5d2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
@@ -50,6 +50,7 @@
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.hdfs.util.RwLockMode;
@@ -597,9 +598,13 @@ private int getTrueReplication(MiniDFSCluster cluster, ExtendedBlock block)
       throws IOException {
     int count = 0;
     for (DataNode dn : cluster.getDataNodes()) {
-      if (DataNodeTestUtils.getFSDataset(dn).getStoredBlock(
-          block.getBlockPoolId(), block.getBlockId()) != null) {
-        count++;
+      try {
+        if (DataNodeTestUtils.getFSDataset(dn).getStoredBlock(
+            block.getBlockPoolId(), block.getBlockId()) != null) {
+          count++;
+        }
+      } catch (ReplicaNotFoundException e) {
+        continue;
       }
     }
     return count;

From ead144e9d9dc70e22e1e695294f2717b8c2a1bd8 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Fri, 29 Nov 2024 10:54:26 +0800
Subject: [PATCH 07/22] trigger yetus.


From c75c2412d4a07e7b4854185dfb8ad0a06e4f3733 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Tue, 3 Dec 2024 09:43:24 +0800
Subject: [PATCH 08/22] fix checkstyle

---
 .../org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java   | 3 ++-
 .../hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index 3b3e9444f59df..dc5d70756277c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -133,7 +133,8 @@ public static File idToBlockDir(File root, long blockId) {
    * Take an example. We hava a block with blockid mapping to:
    * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir0"
    * We return "subdir0/subdir0"
-   * @return
+   * @param blockId blockId
+   * @return The two-level subdir name
    */
   public static String idToBlockDirSuffixName(long blockId) {
     int d1 = (int) ((blockId >> 16) & 0x1F);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index fa677f0ceab0d..f58ee729ef98f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -68,7 +68,6 @@
 import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaHandler;
 import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
-import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException;
 import org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry;
 import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;

From 643ab8dff5279a6b7e0bdc3e8afaf6c6a9f1d54c Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Tue, 3 Dec 2024 14:46:01 +0800
Subject: [PATCH 09/22] checkAndUpdate use subdir lock

---
 .../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index c118190827ba6..6598a8c3dca81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -2774,7 +2774,8 @@ public void checkAndUpdate(String bpid, ScanInfo scanInfo)
       lastDirScannerNotifyTime = startTimeMs;
     }
     String storageUuid = vol.getStorageID();
-    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME, bpid, storageUuid)) {
+    try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR, bpid,
+        vol.getStorageID(), DatanodeUtil.idToBlockDirSuffixName(blockId))) {
       if (!storageMap.containsKey(storageUuid)) {
         // Storage was already removed
         return;

From 81745e16811abd33299afc5f38d264d2b73d6b53 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Sun, 8 Dec 2024 18:56:11 +0800
Subject: [PATCH 10/22] fix npe check.

---
 .../hadoop/hdfs/server/datanode/DataSetLockManager.java      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
index 1e58d6b3500a8..dafbb4ed50227 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetLockManager.java
@@ -97,6 +97,11 @@ private String generateLockName(LockLevel level, String... resources) {
       }
       return resources[0] + resources[1];
     } else if (resources.length == 3 && level == LockLevel.DIR) {
+      if (resources[0] == null || resources[1] == null || resources[2] == null) {
+        throw new IllegalArgumentException("acquire a null dataset lock : "
+            + resources[0] + ",volume lock :" + resources[1]
+        + ",subdir lock :" + resources[2]);
+      }
       return resources[0] + resources[1] + resources[2];
     } else {
       throw new IllegalArgumentException("lock level do not match resource");

From a95754b99434d33aa3b32760bbccbd9b9456609a Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Mon, 16 Dec 2024 14:04:35 +0800
Subject: [PATCH 11/22] Add interface DataSetSubLockStrategy

---
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java |  4 ++
 .../datanode/DataSetSubLockStrategy.java      | 36 +++++++++++
 .../hdfs/server/datanode/DatanodeUtil.java    | 27 --------
 .../datanode/ModDataSetSubLockStrategy.java   | 53 ++++++++++++++++
 .../fsdataset/impl/FsDatasetImpl.java         | 62 +++++++++++--------
 5 files changed, 128 insertions(+), 54 deletions(-)
 create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java
 create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index ac8e475c50138..5492c34ad4e59 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1765,6 +1765,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final boolean
       DFS_DATANODE_LOCKMANAGER_TRACE_DEFAULT = false;
 
+  public static final String DFS_DATANODE_DATASET_SUBLOCK_COUNT_KEY =
+      "dfs.datanode.dataset.sublock.count";
+  public static final long DFS_DATANODE_DATASET_SUBLOCK_COUNT_DEFAULT = 1000L;
+
   // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
   @Deprecated
   public static final String  DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java
new file mode 100644
index 0000000000000..7ba1df8df5232
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import java.util.List;
+
+/**
+ * This interface is used to generate sub lock name for a blockid.
+ */
+public interface DataSetSubLockStrategy {
+
+  /**
+   * Generate sub lock name for the given blockid.
+   * @param blockid the block id.
+   * @return sub lock name for the input blockid.
+   */
+  String blockIdToSubLock(long blockid);
+
+  List<String> getAllSubLockName();
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index dc5d70756277c..c98ff5413bd85 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -21,8 +21,6 @@
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -129,31 +127,6 @@ public static File idToBlockDir(File root, long blockId) {
     return new File(root, path);
   }
 
-  /**
-   * Take an example. We hava a block with blockid mapping to:
-   * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir0"
-   * We return "subdir0/subdir0"
-   * @param blockId blockId
-   * @return The two-level subdir name
-   */
-  public static String idToBlockDirSuffixName(long blockId) {
-    int d1 = (int) ((blockId >> 16) & 0x1F);
-    int d2 = (int) ((blockId >> 8) & 0x1F);
-    return DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
-        DataStorage.BLOCK_SUBDIR_PREFIX + d2;
-  }
-
-  public static List<String> getAllSubDirNameForDataSetLock() {
-    List<String> res = new ArrayList<>();
-    for (int d1 = 0; d1 <= 0x1F; d1++) {
-      for (int d2 = 0; d2 <= 0x1F; d2++) {
-        res.add(DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
-            DataStorage.BLOCK_SUBDIR_PREFIX + d2);
-      }
-    }
-    return res;
-  }
-
   /**
    * @return the FileInputStream for the meta data of the given block.
    * @throws FileNotFoundException
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java
new file mode 100644
index 0000000000000..3f22ca7a0b8ca
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ModDataSetSubLockStrategy implements DataSetSubLockStrategy {
+  public static final Logger LOG = LoggerFactory.getLogger(DataSetSubLockStrategy.class);
+
+  private static final String LOCK_NAME_PERFIX = "SubLock";
+  private long modFactor;
+
+  public ModDataSetSubLockStrategy(long mod) {
+    if (mod <= 0) {
+      mod = 1L;
+    }
+    this.modFactor = mod;
+  }
+
+  @Override
+  public String blockIdToSubLock(long blockid) {
+    return LOCK_NAME_PERFIX + String.valueOf(blockid % modFactor);
+  }
+
+  @Override
+  public List<String> getAllSubLockName() {
+    List<String> res = new ArrayList<>();
+    for (long i = 0L; i < modFactor; i++) {
+      res.add(LOCK_NAME_PERFIX + i);
+    }
+    return res;
+  }
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 6598a8c3dca81..91b12daef8143 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -65,9 +65,11 @@
 import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
 import org.apache.hadoop.hdfs.server.datanode.DataSetLockManager;
+import org.apache.hadoop.hdfs.server.datanode.DataSetSubLockStrategy;
 import org.apache.hadoop.hdfs.server.datanode.FileIoProvider;
 import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
 import org.apache.hadoop.hdfs.server.datanode.LocalReplica;
+import org.apache.hadoop.hdfs.server.datanode.ModDataSetSubLockStrategy;
 import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
 import org.apache.hadoop.util.AutoCloseableLock;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -200,7 +202,7 @@ public Block getStoredBlock(String bpid, long blkid)
       throws IOException {
     try (AutoCloseableLock lock = lockManager.readLock(LockLevel.DIR,
         bpid, getReplicaInfo(bpid, blkid).getStorageUuid(),
-        DatanodeUtil.idToBlockDirSuffixName(blkid))) {
+        datasetSubLockStrategy.blockIdToSubLock(blkid))) {
       ReplicaInfo r = volumeMap.get(bpid, blkid);
       if (r == null) {
         return null;
@@ -289,6 +291,9 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
   private long lastDirScannerNotifyTime;
   private volatile long lastDirScannerFinishTime;
 
+  private final DataSetSubLockStrategy datasetSubLockStrategy;
+  private final long datasetSubLockCount;
+
   /**
    * An FSDataset has a directory where it loads its data files.
    */
@@ -393,6 +398,9 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
         DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_MAX_NOTIFY_COUNT_KEY,
         DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_MAX_NOTIFY_COUNT_DEFAULT);
     lastDirScannerNotifyTime = System.currentTimeMillis();
+    datasetSubLockCount = conf.getLong(DFSConfigKeys.DFS_DATANODE_DATASET_SUBLOCK_COUNT_KEY,
+        DFSConfigKeys.DFS_DATANODE_DATASET_SUBLOCK_COUNT_DEFAULT);
+    this.datasetSubLockStrategy = new ModDataSetSubLockStrategy(datasetSubLockCount);
   }
 
   /**
@@ -431,7 +439,7 @@ private synchronized void activateVolume(
       FsVolumeReference ref) throws IOException {
     for (String bp : volumeMap.getBlockPoolList()) {
       lockManager.addLock(LockLevel.VOLUME, bp, ref.getVolume().getStorageID());
-      List<String> allSubDirNameForDataSetLock = DatanodeUtil.getAllSubDirNameForDataSetLock();
+      List<String> allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockName();
       for (String dir : allSubDirNameForDataSetLock) {
         lockManager.addLock(LockLevel.DIR, bp, ref.getVolume().getStorageID(), dir);
         LOG.info("Added DIR lock for bpid:{}, volume storageid:{}, dir:{}",
@@ -636,7 +644,7 @@ public void removeVolumes(
       for (String storageUuid : storageToRemove) {
         storageMap.remove(storageUuid);
         for (String bp : volumeMap.getBlockPoolList()) {
-          List<String> allSubDirNameForDataSetLock = DatanodeUtil.getAllSubDirNameForDataSetLock();
+          List<String> allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockName();
           for (String dir : allSubDirNameForDataSetLock) {
             lockManager.removeLock(LockLevel.DIR, bp, storageUuid, dir);
             LOG.info("Removed DIR lock for bpid:{}, volume storageid:{}, dir:{}",
@@ -834,7 +842,7 @@ public InputStream getBlockInputStream(ExtendedBlock b,
     ReplicaInfo info;
     try (AutoCloseableLock lock = lockManager.readLock(LockLevel.DIR,
         b.getBlockPoolId(), getStorageUuidForLock(b),
-        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
       info = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock());
     }
 
@@ -930,7 +938,7 @@ public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b,
       long blkOffset, long metaOffset) throws IOException {
     try (AutoCloseDataSetLock l = lockManager.readLock(LockLevel.DIR,
         b.getBlockPoolId(), getStorageUuidForLock(b),
-        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
       ReplicaInfo info = getReplicaInfo(b);
       FsVolumeReference ref = info.getVolume().obtainReference();
       try {
@@ -1397,7 +1405,7 @@ public ReplicaHandler append(ExtendedBlock b,
       long newGS, long expectedBlockLen) throws IOException {
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
         b.getBlockPoolId(), getStorageUuidForLock(b),
-        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
       // If the block was successfully finalized because all packets
       // were successfully processed at the Datanode but the ack for
       // some of the packets were not received by the client. The client
@@ -1451,7 +1459,7 @@ private ReplicaInPipeline append(String bpid,
       throws IOException {
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
         bpid, replicaInfo.getStorageUuid(),
-        DatanodeUtil.idToBlockDirSuffixName(replicaInfo.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(replicaInfo.getBlockId()))) {
       // If the block is cached, start uncaching it.
       if (replicaInfo.getState() != ReplicaState.FINALIZED) {
         throw new IOException("Only a Finalized replica can be appended to; "
@@ -1549,7 +1557,7 @@ public ReplicaHandler recoverAppend(
       try {
         try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
             b.getBlockPoolId(), getStorageUuidForLock(b),
-            DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+            datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
           ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
           FsVolumeReference ref = replicaInfo.getVolume().obtainReference();
           ReplicaInPipeline replica;
@@ -1584,7 +1592,7 @@ public Replica recoverClose(ExtendedBlock b, long newGS,
       try {
         try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
             b.getBlockPoolId(), getStorageUuidForLock(b),
-            DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+            datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
           // check replica's state
           ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
           // bump the replica's GS
@@ -1671,7 +1679,7 @@ public ReplicaHandler createRbw(
       ReplicaInPipeline newReplicaInfo;
       try (AutoCloseableLock l = lockManager.writeLock(LockLevel.DIR,
           b.getBlockPoolId(), v.getStorageID(),
-          DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+          datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
         newReplicaInfo = v.createRbw(b);
         if (newReplicaInfo.getReplicaInfo().getState() != ReplicaState.RBW) {
           throw new IOException("CreateRBW returned a replica of state "
@@ -1703,7 +1711,7 @@ public ReplicaHandler recoverRbw(
         try {
           try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
               b.getBlockPoolId(), getStorageUuidForLock(b),
-              DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+              datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
             ReplicaInfo replicaInfo =
                 getReplicaInfo(b.getBlockPoolId(), b.getBlockId());
             // check the replica's state
@@ -1736,7 +1744,7 @@ private ReplicaHandler recoverRbwImpl(ReplicaInPipeline rbw,
       throws IOException {
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
         b.getBlockPoolId(), getStorageUuidForLock(b),
-        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
       // check generation stamp
       long replicaGenerationStamp = rbw.getGenerationStamp();
       if (replicaGenerationStamp < b.getGenerationStamp() ||
@@ -1799,7 +1807,7 @@ public ReplicaInPipeline convertTemporaryToRbw(
     long startTimeMs = Time.monotonicNow();
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
         b.getBlockPoolId(), getStorageUuidForLock(b),
-        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
       final long blockId = b.getBlockId();
       final long expectedGs = b.getGenerationStamp();
       final long visible = b.getNumBytes();
@@ -1940,7 +1948,7 @@ public ReplicaHandler createTemporary(StorageType storageType,
     ReplicaInPipeline newReplicaInfo;
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
         b.getBlockPoolId(), v.getStorageID(),
-        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
       try {
         newReplicaInfo = v.createTemporary(b);
         LOG.debug("creating temporary for block: {} on volume: {}",
@@ -1999,7 +2007,7 @@ public void finalizeBlock(ExtendedBlock b, boolean fsyncDir)
     long startTimeMs = Time.monotonicNow();
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
         b.getBlockPoolId(), getStorageUuidForLock(b),
-        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
       if (Thread.interrupted()) {
         // Don't allow data modifications from interrupted threads
         throw new IOException("Cannot finalize block: " + b + " from Interrupted Thread");
@@ -2037,7 +2045,7 @@ private ReplicaInfo finalizeReplica(String bpid, ReplicaInfo replicaInfo)
       throws IOException {
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
         bpid, replicaInfo.getStorageUuid(),
-        DatanodeUtil.idToBlockDirSuffixName(replicaInfo.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(replicaInfo.getBlockId()))) {
       // Compare generation stamp of old and new replica before finalizing
       if (volumeMap.get(bpid, replicaInfo.getBlockId()).getGenerationStamp()
           > replicaInfo.getGenerationStamp()) {
@@ -2088,7 +2096,7 @@ public void unfinalizeBlock(ExtendedBlock b) throws IOException {
     long startTimeMs = Time.monotonicNow();
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR,
         b.getBlockPoolId(), getStorageUuidForLock(b),
-        DatanodeUtil.idToBlockDirSuffixName(b.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(b.getBlockId()))) {
       ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(),
           b.getLocalBlock());
       if (replicaInfo != null &&
@@ -2487,7 +2495,7 @@ boolean removeReplicaFromMem(final ExtendedBlock block, final FsVolumeImpl volum
     final Block localBlock = block.getLocalBlock();
     final long blockId = localBlock.getBlockId();
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR, bpid, volume.getStorageID(),
-        DatanodeUtil.idToBlockDirSuffixName(blockId))) {
+        datasetSubLockStrategy.blockIdToSubLock(blockId))) {
       final ReplicaInfo info = volumeMap.get(bpid, localBlock);
       if (info == null) {
         ReplicaInfo infoByBlockId = volumeMap.get(bpid, blockId);
@@ -2577,7 +2585,7 @@ private void cacheBlock(String bpid, long blockId) {
       return;
     }
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR, bpid,
-        info.getStorageUuid(), DatanodeUtil.idToBlockDirSuffixName(blockId))) {
+        info.getStorageUuid(), datasetSubLockStrategy.blockIdToSubLock(blockId))) {
       boolean success = false;
       try {
         info = volumeMap.get(bpid, blockId);
@@ -2775,7 +2783,7 @@ public void checkAndUpdate(String bpid, ScanInfo scanInfo)
     }
     String storageUuid = vol.getStorageID();
     try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.DIR, bpid,
-        vol.getStorageID(), DatanodeUtil.idToBlockDirSuffixName(blockId))) {
+        vol.getStorageID(), datasetSubLockStrategy.blockIdToSubLock(blockId))) {
       if (!storageMap.containsKey(storageUuid)) {
         // Storage was already removed
         return;
@@ -3031,8 +3039,8 @@ ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map,
         }
         LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId
             + ", replica=" + replica);
-        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.DIR, bpid,
-            replica.getStorageUuid(), DatanodeUtil.idToBlockDirSuffixName(block.getBlockId()))) {
+        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.VOLUME, bpid,
+            replica.getStorageUuid())) {
           return initReplicaRecoveryImpl(bpid, map, block, recoveryId);
         }
       } catch (MustStopExistingWriter e) {
@@ -3053,8 +3061,8 @@ static ReplicaRecoveryInfo initReplicaRecovery(String bpid, ReplicaMap map,
         }
         LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId
             + ", replica=" + replica);
-        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.DIR, bpid,
-            replica.getStorageUuid(), DatanodeUtil.idToBlockDirSuffixName(block.getBlockId()))) {
+        try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.VOLUME, bpid,
+            replica.getStorageUuid())) {
           return initReplicaRecoveryImpl(bpid, map, block, recoveryId);
         }
       } catch (MustStopExistingWriter e) {
@@ -3262,7 +3270,7 @@ public long getReplicaVisibleLength(final ExtendedBlock block)
   throws IOException {
     try (AutoCloseableLock lock = lockManager.readLock(LockLevel.DIR,
         block.getBlockPoolId(), getStorageUuidForLock(block),
-        DatanodeUtil.idToBlockDirSuffixName(block.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(block.getBlockId()))) {
       final Replica replica = getReplicaInfo(block.getBlockPoolId(),
           block.getBlockId());
       if (replica.getGenerationStamp() < block.getGenerationStamp()) {
@@ -3289,7 +3297,7 @@ public void addBlockPool(String bpid, Configuration conf)
       Set<String> vols = storageMap.keySet();
       for (String v : vols) {
         lockManager.addLock(LockLevel.VOLUME, bpid, v);
-        List<String> allSubDirNameForDataSetLock = DatanodeUtil.getAllSubDirNameForDataSetLock();
+        List<String> allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockName();
         for (String dir : allSubDirNameForDataSetLock) {
           lockManager.addLock(LockLevel.DIR, bpid, v, dir);
           LOG.info("Added DIR lock for bpid:{}, volume storageid:{}, dir:{}",
@@ -3424,7 +3432,7 @@ public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block)
       throws IOException {
     try (AutoCloseableLock lock = lockManager.readLock(LockLevel.DIR,
         block.getBlockPoolId(), getStorageUuidForLock(block),
-        DatanodeUtil.idToBlockDirSuffixName(block.getBlockId()))) {
+        datasetSubLockStrategy.blockIdToSubLock(block.getBlockId()))) {
       final Replica replica = volumeMap.get(block.getBlockPoolId(),
           block.getBlockId());
       if (replica == null) {

From 31910844b23154bd60fdc1d8bbbf516525110ac8 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Tue, 24 Dec 2024 10:15:58 +0800
Subject: [PATCH 12/22] fix failed UT.

---
 .../hdfs/server/datanode/ModDataSetSubLockStrategy.java  | 2 +-
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml      | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java
index 3f22ca7a0b8ca..5e736e54716fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java
@@ -39,7 +39,7 @@ public ModDataSetSubLockStrategy(long mod) {
 
   @Override
   public String blockIdToSubLock(long blockid) {
-    return LOCK_NAME_PERFIX + String.valueOf(blockid % modFactor);
+    return LOCK_NAME_PERFIX + (blockid % modFactor);
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 2e1563444a245..0626216689dac 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -6631,6 +6631,15 @@
       problem. In produce default set false, because it's have little performance loss.
     </description>
   </property>
+
+  <property>
+    <name>dfs.datanode.dataset.sublock.count</name>
+    <value>1000</value>
+    <description>
+      The dataset readwrite lock counts for a volume.
+    </description>
+  </property>
+
   <property>
     <name>dfs.client.fsck.connect.timeout</name>
     <value>60000ms</value>

From 5a78466a2de51255bce2a58fcafdf7ebb04d7460 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Thu, 9 Jan 2025 16:05:41 +0800
Subject: [PATCH 13/22] remove ModDataSetSubLockStrategy, add
 DataNodeLayoutSubLockStrategy

---
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java |  4 ---
 ...ava => DataNodeLayoutSubLockStrategy.java} | 26 ++---------------
 .../hdfs/server/datanode/DatanodeUtil.java    | 28 +++++++++++++++++++
 .../fsdataset/impl/FsDatasetImpl.java         |  7 ++---
 .../src/main/resources/hdfs-default.xml       |  8 ------
 5 files changed, 33 insertions(+), 40 deletions(-)
 rename hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/{ModDataSetSubLockStrategy.java => DataNodeLayoutSubLockStrategy.java} (60%)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 5492c34ad4e59..ac8e475c50138 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -1765,10 +1765,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final boolean
       DFS_DATANODE_LOCKMANAGER_TRACE_DEFAULT = false;
 
-  public static final String DFS_DATANODE_DATASET_SUBLOCK_COUNT_KEY =
-      "dfs.datanode.dataset.sublock.count";
-  public static final long DFS_DATANODE_DATASET_SUBLOCK_COUNT_DEFAULT = 1000L;
-
   // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
   @Deprecated
   public static final String  DFS_CLIENT_RETRY_POLICY_ENABLED_KEY
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java
similarity index 60%
rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java
rename to hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java
index 5e736e54716fd..3880b991e8dca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ModDataSetSubLockStrategy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java
@@ -18,36 +18,16 @@
 
 package org.apache.hadoop.hdfs.server.datanode;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
 import java.util.List;
 
-public class ModDataSetSubLockStrategy implements DataSetSubLockStrategy {
-  public static final Logger LOG = LoggerFactory.getLogger(DataSetSubLockStrategy.class);
-
-  private static final String LOCK_NAME_PERFIX = "SubLock";
-  private long modFactor;
-
-  public ModDataSetSubLockStrategy(long mod) {
-    if (mod <= 0) {
-      mod = 1L;
-    }
-    this.modFactor = mod;
-  }
-
+public class DataNodeLayoutSubLockStrategy implements DataSetSubLockStrategy {
   @Override
   public String blockIdToSubLock(long blockid) {
-    return LOCK_NAME_PERFIX + (blockid % modFactor);
+    return DatanodeUtil.idToBlockDirSuffixName(blockid);
   }
 
   @Override
   public List<String> getAllSubLockName() {
-    List<String> res = new ArrayList<>();
-    for (long i = 0L; i < modFactor; i++) {
-      res.add(LOCK_NAME_PERFIX + i);
-    }
-    return res;
+    return DatanodeUtil.getAllSubDirNameForDataSetLock();
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index c98ff5413bd85..ab3d8ab96cacb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -21,6 +21,8 @@
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -115,6 +117,8 @@ public static boolean dirNoFilesRecursive(
   /**
    * Get the directory where a finalized block with this ID should be stored.
    * Do not attempt to create the directory.
+   * Note: update {@link DatanodeUtil#idToBlockDirSuffixName(long)} and
+   * {@link DatanodeUtil#getAllSubDirNameForDataSetLock()} when current method changed.
    * @param root the root directory where finalized blocks are stored
    * @param blockId
    * @return
@@ -127,6 +131,30 @@ public static File idToBlockDir(File root, long blockId) {
     return new File(root, path);
   }
 
+  /**
+   * Take an example: we hava a block with blockid mapping to:
+   * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir0"
+   * We return "subdir0/subdir0"
+   * @return
+   */
+  public static String idToBlockDirSuffixName(long blockId) {
+    int d1 = (int) ((blockId >> 16) & 0x1F);
+    int d2 = (int) ((blockId >> 8) & 0x1F);
+    return DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
+        DataStorage.BLOCK_SUBDIR_PREFIX + d2;
+  }
+
+  public static List<String> getAllSubDirNameForDataSetLock() {
+    List<String> res = new ArrayList<>();
+    for (int d1 = 0; d1 <= 0x1F; d1++) {
+      for (int d2 = 0; d2 <= 0x1F; d2++) {
+        res.add(DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
+            DataStorage.BLOCK_SUBDIR_PREFIX + d2);
+      }
+    }
+    return res;
+  }
+
   /**
    * @return the FileInputStream for the meta data of the given block.
    * @throws FileNotFoundException
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 91b12daef8143..e9709a151d8e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -64,12 +64,12 @@
 import org.apache.hadoop.hdfs.server.common.DataNodeLockManager;
 import org.apache.hadoop.hdfs.server.common.DataNodeLockManager.LockLevel;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutSubLockStrategy;
 import org.apache.hadoop.hdfs.server.datanode.DataSetLockManager;
 import org.apache.hadoop.hdfs.server.datanode.DataSetSubLockStrategy;
 import org.apache.hadoop.hdfs.server.datanode.FileIoProvider;
 import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
 import org.apache.hadoop.hdfs.server.datanode.LocalReplica;
-import org.apache.hadoop.hdfs.server.datanode.ModDataSetSubLockStrategy;
 import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
 import org.apache.hadoop.util.AutoCloseableLock;
 import org.apache.hadoop.hdfs.protocol.Block;
@@ -292,7 +292,6 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
   private volatile long lastDirScannerFinishTime;
 
   private final DataSetSubLockStrategy datasetSubLockStrategy;
-  private final long datasetSubLockCount;
 
   /**
    * An FSDataset has a directory where it loads its data files.
@@ -398,9 +397,7 @@ public LengthInputStream getMetaDataInputStream(ExtendedBlock b)
         DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_MAX_NOTIFY_COUNT_KEY,
         DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_MAX_NOTIFY_COUNT_DEFAULT);
     lastDirScannerNotifyTime = System.currentTimeMillis();
-    datasetSubLockCount = conf.getLong(DFSConfigKeys.DFS_DATANODE_DATASET_SUBLOCK_COUNT_KEY,
-        DFSConfigKeys.DFS_DATANODE_DATASET_SUBLOCK_COUNT_DEFAULT);
-    this.datasetSubLockStrategy = new ModDataSetSubLockStrategy(datasetSubLockCount);
+    this.datasetSubLockStrategy = new DataNodeLayoutSubLockStrategy();
   }
 
   /**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0626216689dac..1ed63183917ed 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -6632,14 +6632,6 @@
     </description>
   </property>
 
-  <property>
-    <name>dfs.datanode.dataset.sublock.count</name>
-    <value>1000</value>
-    <description>
-      The dataset readwrite lock counts for a volume.
-    </description>
-  </property>
-
   <property>
     <name>dfs.client.fsck.connect.timeout</name>
     <value>60000ms</value>

From a17e76b45c1ea71472f5bdd3ff852063aa0b6632 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Thu, 9 Jan 2025 16:13:01 +0800
Subject: [PATCH 14/22] fix blank line.

---
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml              | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 1ed63183917ed..2e1563444a245 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -6631,7 +6631,6 @@
       problem. In produce default set false, because it's have little performance loss.
     </description>
   </property>
-
   <property>
     <name>dfs.client.fsck.connect.timeout</name>
     <value>60000ms</value>

From ed853bc4d8109ddffc1e1474d31020e4410ce6ff Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Thu, 9 Jan 2025 20:53:38 +0800
Subject: [PATCH 15/22] fix checkstyle,

---
 .../apache/hadoop/hdfs/server/datanode/DatanodeUtil.java   | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index ab3d8ab96cacb..07edaa12125de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -132,10 +132,11 @@ public static File idToBlockDir(File root, long blockId) {
   }
 
   /**
-   * Take an example: we hava a block with blockid mapping to:
+   * Take an example.
+   * We hava a block with blockid mapping to:
    * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir0"
-   * We return "subdir0/subdir0"
-   * @return
+   * We return "subdir0/subdir0".
+   * @return two-level subdir string where block will be stored.
    */
   public static String idToBlockDirSuffixName(long blockId) {
     int d1 = (int) ((blockId >> 16) & 0x1F);

From e5e7086eea51cd7906b61d2107114336b6f18ef2 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Thu, 9 Jan 2025 20:56:54 +0800
Subject: [PATCH 16/22] fix javadoc.

---
 .../org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index 07edaa12125de..78dce91f14b14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -136,6 +136,7 @@ public static File idToBlockDir(File root, long blockId) {
    * We hava a block with blockid mapping to:
    * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir0"
    * We return "subdir0/subdir0".
+   * @param blockId the block id.
    * @return two-level subdir string where block will be stored.
    */
   public static String idToBlockDirSuffixName(long blockId) {

From bfb369369ca164de346d9836edfb87ba3bb59813 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Mon, 13 Jan 2025 20:47:44 +0800
Subject: [PATCH 17/22] trigger yetus.


From e533c20b44675052e1d3ee5bb1c3676517929a36 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Thu, 16 Jan 2025 16:07:07 +0800
Subject: [PATCH 18/22] refactor DatanodeUtil#idToBlockDir

---
 .../DataNodeLayoutSubLockStrategy.java        |  2 +-
 .../hdfs/server/datanode/DatanodeUtil.java    | 41 +++++++++----------
 2 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java
index 3880b991e8dca..98df7015d17a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java
@@ -23,7 +23,7 @@
 public class DataNodeLayoutSubLockStrategy implements DataSetSubLockStrategy {
   @Override
   public String blockIdToSubLock(long blockid) {
-    return DatanodeUtil.idToBlockDirSuffixName(blockid);
+    return DatanodeUtil.idToBlockDirSuffix(blockid);
   }
 
   @Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index 78dce91f14b14..4b8012635114f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -39,6 +39,8 @@ public class DatanodeUtil {
   public static final String DISK_ERROR = "Possible disk error: ";
 
   private static final String SEP = System.getProperty("file.separator");
+  
+  private static final long MASK = 0x1F;
 
   /** Get the cause of an I/O exception if caused by a possible disk error
    * @param ioe an I/O exception
@@ -114,23 +116,6 @@ public static boolean dirNoFilesRecursive(
     return true;
   }
 
-  /**
-   * Get the directory where a finalized block with this ID should be stored.
-   * Do not attempt to create the directory.
-   * Note: update {@link DatanodeUtil#idToBlockDirSuffixName(long)} and
-   * {@link DatanodeUtil#getAllSubDirNameForDataSetLock()} when current method changed.
-   * @param root the root directory where finalized blocks are stored
-   * @param blockId
-   * @return
-   */
-  public static File idToBlockDir(File root, long blockId) {
-    int d1 = (int) ((blockId >> 16) & 0x1F);
-    int d2 = (int) ((blockId >> 8) & 0x1F);
-    String path = DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
-        DataStorage.BLOCK_SUBDIR_PREFIX + d2;
-    return new File(root, path);
-  }
-
   /**
    * Take an example.
    * We hava a block with blockid mapping to:
@@ -139,17 +124,29 @@ public static File idToBlockDir(File root, long blockId) {
    * @param blockId the block id.
    * @return two-level subdir string where block will be stored.
    */
-  public static String idToBlockDirSuffixName(long blockId) {
-    int d1 = (int) ((blockId >> 16) & 0x1F);
-    int d2 = (int) ((blockId >> 8) & 0x1F);
+  public static String idToBlockDirSuffix(long blockId) {
+    int d1 = (int) ((blockId >> 16) & MASK);
+    int d2 = (int) ((blockId >> 8) & MASK);
     return DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
         DataStorage.BLOCK_SUBDIR_PREFIX + d2;
   }
 
+  /**
+   * Get the directory where a finalized block with this ID should be stored.
+   * Do not attempt to create the directory.
+   * @param root the root directory where finalized blocks are stored
+   * @param blockId
+   * @return
+   */
+  public static File idToBlockDir(File root, long blockId) {
+    String path = idToBlockDirSuffix(blockId);
+    return new File(root, path);
+  }
+
   public static List<String> getAllSubDirNameForDataSetLock() {
     List<String> res = new ArrayList<>();
-    for (int d1 = 0; d1 <= 0x1F; d1++) {
-      for (int d2 = 0; d2 <= 0x1F; d2++) {
+    for (int d1 = 0; d1 <= MASK; d1++) {
+      for (int d2 = 0; d2 <= MASK; d2++) {
         res.add(DataStorage.BLOCK_SUBDIR_PREFIX + d1 + SEP +
             DataStorage.BLOCK_SUBDIR_PREFIX + d2);
       }

From f585173b1c35fe70e5b879c6ab36937eabba60a4 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Thu, 16 Jan 2025 18:05:50 +0800
Subject: [PATCH 19/22] optimize javadoc.

---
 .../hadoop/hdfs/server/common/DataNodeLockManager.java      | 6 +++++-
 .../hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java | 2 +-
 .../hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java | 2 +-
 .../apache/hadoop/hdfs/server/datanode/DatanodeUtil.java    | 2 +-
 .../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java  | 6 +++---
 5 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java
index cb22a0570627f..49e8b626efbc3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/DataNodeLockManager.java
@@ -24,8 +24,12 @@
 public interface DataNodeLockManager<T extends AutoCloseDataSetLock> {
 
   /**
-   * Acquire block pool level first if you want to Acquire volume lock.
+   * Acquire block pool level and volume level lock first if you want to acquire dir lock.
    * Or only acquire block pool level lock.
+   * There are several locking sequential patterns as below:
+   * 1. block pool
+   * 2. block poll -> volume
+   * 3. block pool level -> volume -> dir
    */
   enum LockLevel {
     BLOCK_POOl,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java
index 98df7015d17a3..7665d9c1cf354 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutSubLockStrategy.java
@@ -27,7 +27,7 @@ public String blockIdToSubLock(long blockid) {
   }
 
   @Override
-  public List<String> getAllSubLockName() {
+  public List<String> getAllSubLockNames() {
     return DatanodeUtil.getAllSubDirNameForDataSetLock();
   }
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java
index 7ba1df8df5232..f5f09882c18e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataSetSubLockStrategy.java
@@ -32,5 +32,5 @@ public interface DataSetSubLockStrategy {
    */
   String blockIdToSubLock(long blockid);
 
-  List<String> getAllSubLockName();
+  List<String> getAllSubLockNames();
 }
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index 4b8012635114f..d39b78097f463 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -119,7 +119,7 @@ public static boolean dirNoFilesRecursive(
   /**
    * Take an example.
    * We hava a block with blockid mapping to:
-   * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir0"
+   * "/data1/hadoop/hdfs/datanode/current/BP-xxxx/current/finalized/subdir0/subdir1"
    * We return "subdir0/subdir0".
    * @param blockId the block id.
    * @return two-level subdir string where block will be stored.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index e9709a151d8e1..934c5faee2f92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -436,7 +436,7 @@ private synchronized void activateVolume(
       FsVolumeReference ref) throws IOException {
     for (String bp : volumeMap.getBlockPoolList()) {
       lockManager.addLock(LockLevel.VOLUME, bp, ref.getVolume().getStorageID());
-      List<String> allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockName();
+      List<String> allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockNames();
       for (String dir : allSubDirNameForDataSetLock) {
         lockManager.addLock(LockLevel.DIR, bp, ref.getVolume().getStorageID(), dir);
         LOG.info("Added DIR lock for bpid:{}, volume storageid:{}, dir:{}",
@@ -641,7 +641,7 @@ public void removeVolumes(
       for (String storageUuid : storageToRemove) {
         storageMap.remove(storageUuid);
         for (String bp : volumeMap.getBlockPoolList()) {
-          List<String> allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockName();
+          List<String> allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockNames();
           for (String dir : allSubDirNameForDataSetLock) {
             lockManager.removeLock(LockLevel.DIR, bp, storageUuid, dir);
             LOG.info("Removed DIR lock for bpid:{}, volume storageid:{}, dir:{}",
@@ -3294,7 +3294,7 @@ public void addBlockPool(String bpid, Configuration conf)
       Set<String> vols = storageMap.keySet();
       for (String v : vols) {
         lockManager.addLock(LockLevel.VOLUME, bpid, v);
-        List<String> allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockName();
+        List<String> allSubDirNameForDataSetLock = datasetSubLockStrategy.getAllSubLockNames();
         for (String dir : allSubDirNameForDataSetLock) {
           lockManager.addLock(LockLevel.DIR, bpid, v, dir);
           LOG.info("Added DIR lock for bpid:{}, volume storageid:{}, dir:{}",

From 81374dc7733b7a542c46c438ee10ced49934d84b Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Thu, 16 Jan 2025 20:42:54 +0800
Subject: [PATCH 20/22] fix blanks eol

---
 .../org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index d39b78097f463..0ef1835d09f55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -39,7 +39,7 @@ public class DatanodeUtil {
   public static final String DISK_ERROR = "Possible disk error: ";
 
   private static final String SEP = System.getProperty("file.separator");
-  
+
   private static final long MASK = 0x1F;
 
   /** Get the cause of an I/O exception if caused by a possible disk error

From 5ceae9211fda9b3a6857d8d1543e8fbce9ccf6d8 Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Fri, 17 Jan 2025 08:05:59 +0800
Subject: [PATCH 21/22] fix checkstyle

---
 .../org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
index 0ef1835d09f55..4961eeba51c23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeUtil.java
@@ -39,7 +39,6 @@ public class DatanodeUtil {
   public static final String DISK_ERROR = "Possible disk error: ";
 
   private static final String SEP = System.getProperty("file.separator");
-
   private static final long MASK = 0x1F;
 
   /** Get the cause of an I/O exception if caused by a possible disk error

From ae9b20b2a4cf5a459bf36b2aecddc142f17dab4b Mon Sep 17 00:00:00 2001
From: zhanghaobo <hfutzhanghb@163.com>
Date: Fri, 14 Feb 2025 09:50:31 +0800
Subject: [PATCH 22/22] trigger yetus.