From c9f0ac3cd95c7843ead69a9d5127c47dfc03eb54 Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Wed, 12 Aug 2020 00:26:22 -0700 Subject: [PATCH 1/4] Patch NamenodeWebHdfsMethods#getTrashRoot. Change-Id: I2f4d818b342add60fd0da6514196237f95046f06 --- .../web/resources/NamenodeWebHdfsMethods.java | 52 +++++++++++++++++-- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 2423a037c8fd0..6543f9ac64bd8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -1345,19 +1345,61 @@ protected Response get( } } + /** + * Get the snapshot root of a given file or directory if it exists. + * e.g. if /snapdir1 is a snapshottable directory and path given is + * /snapdir1/path/to/file, this method would return /snapdir1 + * @param pathStr String of path to a file or a directory. + * @return Not null if found in a snapshot root directory. + * @throws IOException + */ + String getSnapshotRoot(String pathStr) throws IOException { + SnapshottableDirectoryStatus[] dirStatusList = + getRpcClientProtocol().getSnapshottableDirListing(); + if (dirStatusList == null) { + return null; + } + for (SnapshottableDirectoryStatus dirStatus : dirStatusList) { + String currDir = dirStatus.getFullPath().toString(); + if (pathStr.startsWith(currDir)) { + return currDir; + } + } + return null; + } + private String getTrashRoot(Configuration conf, String fullPath) throws IOException { - UserGroupInformation ugi= UserGroupInformation.getCurrentUser(); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String parentSrc = getParent(fullPath); + String ssTrashRoot = null; + boolean isSnapshotTrashRootEnabled = getRpcClientProtocol() + .getServerDefaults().getSnapshotTrashRootEnabled(); + if (isSnapshotTrashRootEnabled) { + String ssRoot = getSnapshotRoot(fullPath); + ssTrashRoot = DFSUtilClient.getSnapshotTrashRoot(ssRoot, ugi); + } EncryptionZone ez = getRpcClientProtocol().getEZForPath( parentSrc != null ? parentSrc : fullPath); - String trashRoot; + String ezTrashRoot = null; if (ez != null) { - trashRoot = DFSUtilClient.getEZTrashRoot(ez, ugi); + ezTrashRoot = DFSUtilClient.getEZTrashRoot(ez, ugi); + } + // Choose the longest path + if (ssTrashRoot == null) { + if (ezTrashRoot == null) { + return DFSUtilClient.getTrashRoot(conf, ugi); + } else { + return ezTrashRoot; + } } else { - trashRoot = DFSUtilClient.getTrashRoot(conf, ugi); + if (ezTrashRoot == null) { + return ssTrashRoot; + } else { + return ssTrashRoot.length() > ezTrashRoot.length() ? + ssTrashRoot : ezTrashRoot; + } } - return trashRoot; } /** From f6da4aa28d8570ad786150ee291fcd50d1708da0 Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Wed, 12 Aug 2020 00:31:50 -0700 Subject: [PATCH 2/4] Improve path selection logic readability. Change-Id: Id6469f43ace030d62a0da6e5d6c4cc689babbfe3 --- .../web/resources/NamenodeWebHdfsMethods.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 6543f9ac64bd8..7d12ad104adb7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -1372,7 +1372,7 @@ private String getTrashRoot(Configuration conf, String fullPath) throws IOException { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String parentSrc = getParent(fullPath); - String ssTrashRoot = null; + String ssTrashRoot = ""; boolean isSnapshotTrashRootEnabled = getRpcClientProtocol() .getServerDefaults().getSnapshotTrashRootEnabled(); if (isSnapshotTrashRootEnabled) { @@ -1381,24 +1381,16 @@ private String getTrashRoot(Configuration conf, String fullPath) } EncryptionZone ez = getRpcClientProtocol().getEZForPath( parentSrc != null ? parentSrc : fullPath); - String ezTrashRoot = null; + String ezTrashRoot = ""; if (ez != null) { ezTrashRoot = DFSUtilClient.getEZTrashRoot(ez, ugi); } // Choose the longest path - if (ssTrashRoot == null) { - if (ezTrashRoot == null) { - return DFSUtilClient.getTrashRoot(conf, ugi); - } else { - return ezTrashRoot; - } + if (ssTrashRoot.isEmpty() && ezTrashRoot.isEmpty()) { + return DFSUtilClient.getTrashRoot(conf, ugi); } else { - if (ezTrashRoot == null) { - return ssTrashRoot; - } else { - return ssTrashRoot.length() > ezTrashRoot.length() ? - ssTrashRoot : ezTrashRoot; - } + return ssTrashRoot.length() > ezTrashRoot.length() ? + ssTrashRoot : ezTrashRoot; } } From 903704c91e0b674e3925fdb8f9c8e1449005b696 Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Wed, 12 Aug 2020 00:41:27 -0700 Subject: [PATCH 3/4] ssRoot could return null. Change-Id: I7845ae4df5233eba5dd2b6245b53a457f8e68446 --- .../server/namenode/web/resources/NamenodeWebHdfsMethods.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 7d12ad104adb7..9baed4f06730c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -1377,7 +1377,9 @@ private String getTrashRoot(Configuration conf, String fullPath) .getServerDefaults().getSnapshotTrashRootEnabled(); if (isSnapshotTrashRootEnabled) { String ssRoot = getSnapshotRoot(fullPath); - ssTrashRoot = DFSUtilClient.getSnapshotTrashRoot(ssRoot, ugi); + if (ssRoot != null) { + ssTrashRoot = DFSUtilClient.getSnapshotTrashRoot(ssRoot, ugi); + } } EncryptionZone ez = getRpcClientProtocol().getEZForPath( parentSrc != null ? parentSrc : fullPath); From 904f104bab990693634b026493a4a4270ff7541b Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Wed, 12 Aug 2020 00:42:01 -0700 Subject: [PATCH 4/4] Add testGetSnapshotTrashRoot. Change-Id: I9fe94a3290e3cab84aa852aa01c9dc104c10d065 --- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 69a0e600ffb62..3c8a92eee12d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -1569,6 +1569,34 @@ public void testGetTrashRoot() throws Exception { assertEquals(expectedPath.toUri().getPath(), trashPath.toUri().getPath()); } + @Test + public void testGetSnapshotTrashRoot() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + conf.setBoolean("dfs.namenode.snapshot.trashroot.enabled", true); + final String currentUser = + UserGroupInformation.getCurrentUser().getShortUserName(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsConstants.WEBHDFS_SCHEME); + Path ssDir1 = new Path("/ssDir1"); + assertTrue(webFS.mkdirs(ssDir1)); + + Path trashPath = webFS.getTrashRoot(ssDir1); + Path expectedPath = new Path(FileSystem.USER_HOME_PREFIX, + new Path(currentUser, FileSystem.TRASH_PREFIX)); + assertEquals(expectedPath.toUri().getPath(), trashPath.toUri().getPath()); + // Enable snapshot + webFS.allowSnapshot(ssDir1); + Path trashPathAfter = webFS.getTrashRoot(ssDir1); + Path expectedPathAfter = new Path(ssDir1, + new Path(FileSystem.TRASH_PREFIX, currentUser)); + assertEquals(expectedPathAfter.toUri().getPath(), + trashPathAfter.toUri().getPath()); + // Cleanup + webFS.disallowSnapshot(ssDir1); + webFS.delete(ssDir1, true); + } + @Test public void testGetEZTrashRoot() throws Exception { final Configuration conf = WebHdfsTestUtil.createConf();