From 6dbc0000277cc3331416f02ef12594da33413f0d Mon Sep 17 00:00:00 2001 From: huangtianhua Date: Mon, 29 Jun 2020 16:54:13 +0800 Subject: [PATCH] HDFS-15025 Applying NVDIMM storage media to HDFS The non-volatile memory NVDIMM is faster than SSD, it can be used simultaneously with RAM, DISK and SSD. Storing the data of HDFS on NVDIMM directly will get better response rate and reliability. --- .../org/apache/hadoop/fs/StorageType.java | 23 ++++++++++++------- .../hadoop/hdfs/protocol/HdfsConstants.java | 2 ++ .../hdfs/protocolPB/PBHelperClient.java | 4 ++++ .../src/main/proto/hdfs.proto | 1 + .../BlockStoragePolicySuite.java | 6 +++++ .../datanode/fsdataset/FsVolumeSpi.java | 3 +++ .../fsdataset/impl/FsDatasetImpl.java | 4 ++-- .../datanode/fsdataset/impl/FsVolumeImpl.java | 7 +++++- .../apache/hadoop/hdfs/tools/DFSAdmin.java | 3 ++- .../server/datanode/SimulatedFSDataset.java | 5 ++++ .../server/datanode/TestDirectoryScanner.java | 5 ++++ .../extdataset/ExternalVolumeImpl.java | 5 ++++ 12 files changed, 56 insertions(+), 12 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java index 2ecd206a04080..78a05fa41b766 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageType.java @@ -34,13 +34,15 @@ @InterfaceStability.Unstable public enum StorageType { // sorted by the speed of the storage types, from fast to slow - RAM_DISK(true), - SSD(false), - DISK(false), - ARCHIVE(false), - PROVIDED(false); + RAM_DISK(true, true), + NVDIMM(false, true), + SSD(false, false), + DISK(false, false), + ARCHIVE(false, false), + PROVIDED(false, false); private final boolean isTransient; + private final boolean isRAM; public static final StorageType DEFAULT = DISK; @@ -48,14 +50,19 @@ public enum StorageType { private static final StorageType[] VALUES = values(); - StorageType(boolean isTransient) { - this.isTransient = isTransient; + StorageType(boolean isTransient, boolean isRAM) { + this.isTransient = isTransienit; + this.isRAM = isRAM; } public boolean isTransient() { return isTransient; } + public boolean isRAM() { + return isRAM; + } + public boolean supportTypeQuota() { return !isTransient; } @@ -93,4 +100,4 @@ private static List getNonTransientTypes() { } return nonTransientTypes; } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index a025b9bad2e69..e3452f651b4c2 100755 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -37,6 +37,8 @@ public final class HdfsConstants { public static final byte MEMORY_STORAGE_POLICY_ID = 15; public static final String MEMORY_STORAGE_POLICY_NAME = "LAZY_PERSIST"; + public static final byte ALLNVDIMM_STORAGE_POLICY_ID = 14; + public static final String ALLNVDIMM_STORAGE_POLICY_NAME = "ALL_NVDIMM"; public static final byte ALLSSD_STORAGE_POLICY_ID = 12; public static final String ALLSSD_STORAGE_POLICY_NAME = "ALL_SSD"; public static final byte ONESSD_STORAGE_POLICY_ID = 10; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index 9fc302464271d..d31c263aa3d45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -472,6 +472,8 @@ public static StorageTypeProto convertStorageType(StorageType type) { return StorageTypeProto.RAM_DISK; case PROVIDED: return StorageTypeProto.PROVIDED; + case NVDIMM: + return StorageTypeProto.NVDIMM; default: throw new IllegalStateException( "BUG: StorageType not found, type=" + type); @@ -490,6 +492,8 @@ public static StorageType convertStorageType(StorageTypeProto type) { return StorageType.RAM_DISK; case PROVIDED: return StorageType.PROVIDED; + case NVDIMM: + return StorageType.NVDIMM; default: throw new IllegalStateException( "BUG: StorageTypeProto not found, type=" + type); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 82fe329c9ce5e..1523c3906e3b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -221,6 +221,7 @@ enum StorageTypeProto { ARCHIVE = 3; RAM_DISK = 4; PROVIDED = 5; + NVDIMM = 6; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java index d62cb727c6c8d..afb95daacbd41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java @@ -63,6 +63,12 @@ public static BlockStoragePolicySuite createDefaultSuite( new StorageType[]{StorageType.DISK}, new StorageType[]{StorageType.DISK}, true); // Cannot be changed on regular files, but inherited. + final byte allNVDIMMId = HdfsConstants.ALLNVDIMM_STORAGE_POLICY_ID; + policies[allNVDIMMId] = new BlockStoragePolicy(allNVDIMMId, + HdfsConstants.ALLNVDIMM_STORAGE_POLICY_NAME, + new StorageType[]{StorageType.NVDIMM}, + new StorageType[]{StorageType.DISK}, + new StorageType[]{StorageType.DISK}); final byte allssdId = HdfsConstants.StoragePolicy.ALL_SSD.value(); policies[allssdId] = new BlockStoragePolicy(allssdId, HdfsConstants.StoragePolicy.ALL_SSD.name(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java index be978d75e9a34..195343cc35cf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java @@ -77,6 +77,9 @@ public interface FsVolumeSpi /** Returns true if the volume is NOT backed by persistent storage. */ boolean isTransientStorage(); + /** Returns true if the volume is NOT backed by RAM storage. */ + boolean isRAMStorage(); + /** * Reserve disk space for a block (RBW or Re-replicating) * so a writer does not run out of space before the block is full. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index a083012a2cf7a..3b784f130535d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -2286,9 +2286,9 @@ private void cacheBlock(String bpid, long blockId) { ": volume was not an instance of FsVolumeImpl."); return; } - if (volume.isTransientStorage()) { + if (volume.isRAMStorage()) { LOG.warn("Caching not supported on block with id " + blockId + - " since the volume is backed by RAM."); + " since the volume is backed by RAM_DISK or NVDIMM."); return; } success = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 5843c7d6696b7..9fd0d1179cbd3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -193,7 +193,7 @@ public class FsVolumeImpl implements FsVolumeSpi { } protected ThreadPoolExecutor initializeCacheExecutor(File parent) { - if (storageType.isTransient()) { + if (storageType.isRAM()) { return null; } if (dataset.datanode == null) { @@ -533,6 +533,11 @@ public boolean isTransientStorage() { return storageType.isTransient(); } + @Override + public boolean isRAMStorage() { + return storageType.isRAM(); + } + @VisibleForTesting public File getFinalizedDir(String bpid) throws IOException { return getBlockPoolSlice(bpid).getFinalizedDir(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index ec5fa0afddc60..f087d404a3472 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -239,7 +239,8 @@ private static class ClearSpaceQuotaCommand extends DFSAdminCommand { "\t\t- DISK\n" + "\t\t- SSD\n" + "\t\t- ARCHIVE\n" + - "\t\t- PROVIDED"; + "\t\t- PROVIDED\n" + + "\t\t- NVDIMM"; private StorageType type; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 8069b64246bc3..2da78c802fce1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -606,6 +606,11 @@ public boolean isTransientStorage() { return false; } + @Override + public boolean isRAMStorage() { + return false; + } + @Override public void reserveSpaceForReplica(long bytesToReserve) { } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index 25470018a9628..d5e3931dd513c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -942,6 +942,11 @@ public boolean isTransientStorage() { return false; } + @Override + public boolean isRAMStorage() { + return false; + } + @Override public void reserveSpaceForReplica(long bytesToReserve) { } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java index 9d62499eb5a3e..6c8e828f3689e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java @@ -67,6 +67,11 @@ public boolean isTransientStorage() { return false; } + @Override + public boolean isRAMStorage() { + return false; + } + @Override public void reserveSpaceForReplica(long bytesToReserve) { }