From 055ce4bb44ed0d2bb329ad7957b592c7e41846ba Mon Sep 17 00:00:00 2001 From: wangyuxuan Date: Thu, 22 Aug 2019 15:43:00 +0800 Subject: [PATCH 1/3] HDFS-14761. RBF: MountTableResolver cannot invalidate cache correctly --- .../resolver/MountTableResolver.java | 3 ++- .../resolver/TestMountTableResolver.java | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java index 8baa5e22a1d10..c72afc57da50d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java @@ -257,10 +257,11 @@ private void invalidateLocationCache(final String path) { Iterator> it = entries.iterator(); while (it.hasNext()) { Entry entry = it.next(); + String key = entry.getKey(); PathLocation loc = entry.getValue(); String src = loc.getSourcePath(); if (src != null) { - if (isParentEntry(src, path)) { + if (isParentEntry(key, path)) { LOG.debug("Removing {}", src); it.remove(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java index ada2815872755..484f12ce7f27f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java @@ -704,4 +704,26 @@ public void testLocationCache() throws Exception { mountTable.removeEntry("/testlocationcache"); mountTable.removeEntry("/anothertestlocationcache"); } + + @Test + public void testInvalidateCache() throws Exception { + Map map1 = getMountTableEntry("1", "/"); + MountTable entry1 = MountTable.newInstance("/", map1); + + mountTable.addEntry(entry1); + assertEquals("1->/", mountTable.getDestinationForPath("/").toString()); + + assertEquals("1->/testInvalidateCache/foo", mountTable + .getDestinationForPath("/testInvalidateCache/foo").toString()); + PathLocation pl = + mountTable.getDestinationForPath("/testInvalidateCache/foo"); + + Map map2 = getMountTableEntry("2", "/testInvalidateCache"); + MountTable entry2 = MountTable.newInstance("/testInvalidateCache", map2); + mountTable.addEntry(entry2); + assertEquals("2->/testInvalidateCache", + mountTable.getDestinationForPath("/testInvalidateCache").toString()); + assertEquals("2->/testInvalidateCache/foo", mountTable + .getDestinationForPath("/testInvalidateCache/foo").toString()); + } } \ No newline at end of file From 8ac28b7956f2dd9dc81d7cef1de7ac8f6f36bc87 Mon Sep 17 00:00:00 2001 From: wangyuxuan Date: Fri, 23 Aug 2019 11:14:19 +0800 Subject: [PATCH 2/3] Add some comments --- .../federation/resolver/TestMountTableResolver.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java index 484f12ce7f27f..268215bcc0391 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java @@ -707,17 +707,18 @@ public void testLocationCache() throws Exception { @Test public void testInvalidateCache() throws Exception { + // Test if we add a new entry, the cached locations which are children of it + // should be invalidate + // Add the entry 1->/ and ensure cache update correctly Map map1 = getMountTableEntry("1", "/"); MountTable entry1 = MountTable.newInstance("/", map1); - mountTable.addEntry(entry1); assertEquals("1->/", mountTable.getDestinationForPath("/").toString()); - assertEquals("1->/testInvalidateCache/foo", mountTable .getDestinationForPath("/testInvalidateCache/foo").toString()); - PathLocation pl = - mountTable.getDestinationForPath("/testInvalidateCache/foo"); + // Add the entry 2->/testInvalidateCache and ensure the cached location + // under it is invalidated correctly Map map2 = getMountTableEntry("2", "/testInvalidateCache"); MountTable entry2 = MountTable.newInstance("/testInvalidateCache", map2); mountTable.addEntry(entry2); From 209cc9c7d4dff97a13e82c7cf6941e409b1d67e2 Mon Sep 17 00:00:00 2001 From: wangyuxuan Date: Fri, 23 Aug 2019 12:07:26 +0800 Subject: [PATCH 3/3] Make highlevel comments as javadoc --- .../server/federation/resolver/TestMountTableResolver.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java index 268215bcc0391..32c54d9cb738c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java @@ -705,10 +705,12 @@ public void testLocationCache() throws Exception { mountTable.removeEntry("/anothertestlocationcache"); } + /** + * Test if we add a new entry, the cached locations which are children of it + * should be invalidate + */ @Test public void testInvalidateCache() throws Exception { - // Test if we add a new entry, the cached locations which are children of it - // should be invalidate // Add the entry 1->/ and ensure cache update correctly Map map1 = getMountTableEntry("1", "/"); MountTable entry1 = MountTable.newInstance("/", map1);