From 15929f6f6c5761088bc00fb18b7b384b75e35da9 Mon Sep 17 00:00:00 2001 From: Karthik Palanisamy Date: Mon, 2 Mar 2020 11:16:16 -0800 Subject: [PATCH 1/5] HDFS-15201 SnapshotCounter hits MaxSnapshotID limit --- .../server/namenode/snapshot/SnapshotManager.java | 6 +++--- .../server/namenode/snapshot/TestSnapshotManager.java | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 2728420e5f7b9..f28cbbfbff32a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -96,7 +96,7 @@ public class SnapshotManager implements SnapshotStatsMXBean { private final boolean snapshotDiffAllowSnapRootDescendant; private final AtomicInteger numSnapshots = new AtomicInteger(); - private static final int SNAPSHOT_ID_BIT_WIDTH = 24; + private static final int SNAPSHOT_ID_BIT_WIDTH = 31; private boolean allowNestedSnapshots = false; private int snapshotCounter = 0; @@ -541,8 +541,8 @@ public void clearSnapshottableDirs() { * * @return maximum allowable snapshot ID. */ - public int getMaxSnapshotID() { - return ((1 << SNAPSHOT_ID_BIT_WIDTH) - 1); + public int getMaxSnapshotID() { + return ((1 << SNAPSHOT_ID_BIT_WIDTH) - 3); } private ObjectName mxBeanName; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java index 3b4b1baa60f18..7d41ba39bc30d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java @@ -92,4 +92,15 @@ public void testSnapshotLimits() throws Exception { StringUtils.toLowerCase(se.getMessage()).contains("rollover")); } } + + + @Test + public void testValidateSnapshotIDWidth() { + FSDirectory fsdir = mock(FSDirectory.class); + SnapshotManager snapshotManager = new SnapshotManager(new Configuration(), + fsdir); + Assert.assertTrue(snapshotManager. + getMaxSnapshotID() < Snapshot.CURRENT_STATE_ID); + } + } From 558f9547389f6df26f110afa359d6ec496a50d8e Mon Sep 17 00:00:00 2001 From: Karthik Palanisamy Date: Mon, 2 Mar 2020 14:17:27 -0800 Subject: [PATCH 2/5] Set SNAPSHOT_ID_BIT_WIDTH to 28 --- .../hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index f28cbbfbff32a..20da02f31e709 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -96,7 +96,7 @@ public class SnapshotManager implements SnapshotStatsMXBean { private final boolean snapshotDiffAllowSnapRootDescendant; private final AtomicInteger numSnapshots = new AtomicInteger(); - private static final int SNAPSHOT_ID_BIT_WIDTH = 31; + private static final int SNAPSHOT_ID_BIT_WIDTH = 28; private boolean allowNestedSnapshots = false; private int snapshotCounter = 0; @@ -542,7 +542,7 @@ public void clearSnapshottableDirs() { * @return maximum allowable snapshot ID. */ public int getMaxSnapshotID() { - return ((1 << SNAPSHOT_ID_BIT_WIDTH) - 3); + return (1 << SNAPSHOT_ID_BIT_WIDTH); } private ObjectName mxBeanName; From a189ef1bb3488d1f11fdc6f9da4e0c2d2c5ddc97 Mon Sep 17 00:00:00 2001 From: Karthik Palanisamy Date: Wed, 4 Mar 2020 13:12:45 -0800 Subject: [PATCH 3/5] Review fix --- .../hdfs/server/namenode/snapshot/SnapshotManager.java | 2 +- .../hdfs/server/namenode/snapshot/TestSnapshotManager.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 20da02f31e709..82c153e0f1e57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -542,7 +542,7 @@ public void clearSnapshottableDirs() { * @return maximum allowable snapshot ID. */ public int getMaxSnapshotID() { - return (1 << SNAPSHOT_ID_BIT_WIDTH); + return ((1 << SNAPSHOT_ID_BIT_WIDTH) - 2); } private ObjectName mxBeanName; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java index 7d41ba39bc30d..287793f74c008 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java @@ -93,7 +93,10 @@ public void testSnapshotLimits() throws Exception { } } - + /** + * Snapshot is recognized by INODE CURRENT_STATE_ID. + * So maximum allowable snapshotID should be less than CURRENT_STATE_ID + */ @Test public void testValidateSnapshotIDWidth() { FSDirectory fsdir = mock(FSDirectory.class); From 3f6830c10796d2f034991070b077f4b91b8e465a Mon Sep 17 00:00:00 2001 From: Karthik Palanisamy Date: Wed, 4 Mar 2020 13:15:18 -0800 Subject: [PATCH 4/5] Update TestSnapshotManager.java --- .../hdfs/server/namenode/snapshot/TestSnapshotManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java index 287793f74c008..4d6e6f4172c02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java @@ -94,7 +94,7 @@ public void testSnapshotLimits() throws Exception { } /** - * Snapshot is recognized by INODE CURRENT_STATE_ID. + * Snapshot is identified by INODE CURRENT_STATE_ID. * So maximum allowable snapshotID should be less than CURRENT_STATE_ID */ @Test From b9f05e22ee9f119cef83fb4c152ee81f81c6fb56 Mon Sep 17 00:00:00 2001 From: Karthik Palanisamy Date: Thu, 12 Mar 2020 17:04:49 -0700 Subject: [PATCH 5/5] Changed -1 from -2 for getMaxSnapshotID --- .../hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 82c153e0f1e57..30b98b8e86421 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -542,7 +542,7 @@ public void clearSnapshottableDirs() { * @return maximum allowable snapshot ID. */ public int getMaxSnapshotID() { - return ((1 << SNAPSHOT_ID_BIT_WIDTH) - 2); + return ((1 << SNAPSHOT_ID_BIT_WIDTH) - 1); } private ObjectName mxBeanName;