From 306b457c22551d8a987f51617fb93395aa98444d Mon Sep 17 00:00:00 2001 From: likeyao <38782735+Likkey@users.noreply.github.com> Date: Thu, 28 Jul 2022 13:48:08 +0800 Subject: [PATCH 1/8] Update NameNodeResourceChecker.java Add Precondition.checkArgument() for minimumRedundantVolumes to ensure that the value is greater than the number of NameNode storage volumes to avoid never being able to turn off safe mode afterwards. --- .../hadoop/hdfs/server/namenode/NameNodeResourceChecker.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java index a1f7c55612343..f419db3f498dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java @@ -174,6 +174,11 @@ private void addDirToCheck(URI directoryToCheck, boolean required) * otherwise. */ public boolean hasAvailableDiskSpace() { + Preconditions.checkArgument(minimumRedundantVolumes <= volumes.size(), + "The setting for " + DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + + " is " + minimumRedundantVolumes + + " which is less than the total number of existing storage volumes " + volumes.size() + + " and will result in adding resources and still not being able to turn off safe mode."); return NameNodeResourcePolicy.areResourcesAvailable(volumes.values(), minimumRedundantVolumes); } From 4dbcf6d4062eeeebc65386ea639231542d07a81f Mon Sep 17 00:00:00 2001 From: likeyao <38782735+Likkey@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:07:00 +0800 Subject: [PATCH 2/8] Update NameNodeResourceChecker.java --- .../hadoop/hdfs/server/namenode/NameNodeResourceChecker.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java index f419db3f498dc..58d11c660311c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java @@ -35,6 +35,10 @@ import org.apache.hadoop.classification.VisibleForTesting; +import com.google.common.collect.Collections2; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; + /** * * NameNodeResourceChecker provides a method - From b3810875d8ef5c15f54332b5cbe505eada2ec6c0 Mon Sep 17 00:00:00 2001 From: likeyao <38782735+Likkey@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:10:08 +0800 Subject: [PATCH 3/8] Update TestNameNodeResourceChecker.java --- .../hdfs/server/namenode/TestNameNodeResourceChecker.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java index f86ce5fc06772..a67de381e5778 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeResourceChecker.java @@ -7,8 +7,10 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * + * * http://www.apache.org/licenses/LICENSE-2.0 * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From 95db4661eae7ac4c62c48dfe85ac485dd8da846f Mon Sep 17 00:00:00 2001 From: likeyao <38782735+Likkey@users.noreply.github.com> Date: Thu, 28 Jul 2022 19:54:24 +0800 Subject: [PATCH 4/8] Update NameNodeResourceChecker.java --- .../hadoop/hdfs/server/namenode/NameNodeResourceChecker.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java index 58d11c660311c..ed06b03a5b0b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java @@ -35,9 +35,7 @@ import org.apache.hadoop.classification.VisibleForTesting; -import com.google.common.collect.Collections2; import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; /** * From 9ca81e12c62e111eccb8c4f8753edbef161d472a Mon Sep 17 00:00:00 2001 From: likeyao <38782735+Likkey@users.noreply.github.com> Date: Fri, 5 Aug 2022 17:06:47 +0800 Subject: [PATCH 5/8] HDFS-16697 --- .../server/namenode/NameNodeResourceChecker.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java index ed06b03a5b0b4..9fcedc2412ff7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java @@ -35,8 +35,6 @@ import org.apache.hadoop.classification.VisibleForTesting; -import com.google.common.base.Preconditions; - /** * * NameNodeResourceChecker provides a method - @@ -176,11 +174,12 @@ private void addDirToCheck(URI directoryToCheck, boolean required) * otherwise. */ public boolean hasAvailableDiskSpace() { - Preconditions.checkArgument(minimumRedundantVolumes <= volumes.size(), - "The setting for " + DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + - " is " + minimumRedundantVolumes + - " which is less than the total number of existing storage volumes " + volumes.size() + - " and will result in adding resources and still not being able to turn off safe mode."); + if (minimumRedundantVolumes > volumes.size()){ + throw new IllegalArgumentException(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + + " is " + minimumRedundantVolumes + + " which is greater than the total number of existing storage volumes " + volumes.size() + + " and will result in adding resources and still not being able to turn off safe mode."); + } return NameNodeResourcePolicy.areResourcesAvailable(volumes.values(), minimumRedundantVolumes); } From ae44e2db8fdd7729f1d2766626482c5d5d236fa7 Mon Sep 17 00:00:00 2001 From: likeyao <38782735+Likkey@users.noreply.github.com> Date: Sat, 6 Aug 2022 16:13:48 +0800 Subject: [PATCH 6/8] Update NameNodeResourceChecker.java --- .../server/namenode/NameNodeResourceChecker.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java index 9fcedc2412ff7..8815453867126 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java @@ -174,11 +174,16 @@ private void addDirToCheck(URI directoryToCheck, boolean required) * otherwise. */ public boolean hasAvailableDiskSpace() { - if (minimumRedundantVolumes > volumes.size()){ - throw new IllegalArgumentException(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + - " is " + minimumRedundantVolumes + - " which is greater than the total number of existing storage volumes " + volumes.size() + - " and will result in adding resources and still not being able to turn off safe mode."); + try{ + if (minimumRedundantVolumes > volumes.size()){ + throw new IllegalArgumentException(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + + " is " + minimumRedundantVolumes + + " which is greater than the total number of existing storage volumes " + volumes.size()); + } + }catch(IllegalArgumentException e){ + LOG.warn(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + + " is greater than the total number of existing storage volumes" + + " and will result in adding resources and still not being able to turn off safe mode.", e); } return NameNodeResourcePolicy.areResourcesAvailable(volumes.values(), minimumRedundantVolumes); From 438bd3edf65d372d2c68e0fca937e1282826ff37 Mon Sep 17 00:00:00 2001 From: likeyao <38782735+Likkey@users.noreply.github.com> Date: Sat, 6 Aug 2022 16:20:36 +0800 Subject: [PATCH 7/8] Update NameNodeResourceChecker.java --- .../hadoop/hdfs/server/namenode/NameNodeResourceChecker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java index 8815453867126..8f1db3c8cdeb6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java @@ -174,13 +174,13 @@ private void addDirToCheck(URI directoryToCheck, boolean required) * otherwise. */ public boolean hasAvailableDiskSpace() { - try{ + try { if (minimumRedundantVolumes > volumes.size()){ throw new IllegalArgumentException(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + " is " + minimumRedundantVolumes + " which is greater than the total number of existing storage volumes " + volumes.size()); } - }catch(IllegalArgumentException e){ + } catch (IllegalArgumentException e){ LOG.warn(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + " is greater than the total number of existing storage volumes" + " and will result in adding resources and still not being able to turn off safe mode.", e); From 130f88ad8f18d7c81c112e54a6e794fa10292713 Mon Sep 17 00:00:00 2001 From: likeyao <38782735+Likkey@users.noreply.github.com> Date: Sat, 6 Aug 2022 16:59:44 +0800 Subject: [PATCH 8/8] Update NameNodeResourceChecker.java --- .../server/namenode/NameNodeResourceChecker.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java index 8f1db3c8cdeb6..bd225e17a40dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeResourceChecker.java @@ -176,14 +176,16 @@ private void addDirToCheck(URI directoryToCheck, boolean required) public boolean hasAvailableDiskSpace() { try { if (minimumRedundantVolumes > volumes.size()){ - throw new IllegalArgumentException(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + - " is " + minimumRedundantVolumes + - " which is greater than the total number of existing storage volumes " + volumes.size()); + throw new IllegalArgumentException("The value of " + + DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + + " is " + minimumRedundantVolumes + + " which is greater than the total number of existing storage volumes " + + volumes.size() + " ."); } } catch (IllegalArgumentException e){ - LOG.warn(DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + - " is greater than the total number of existing storage volumes" + - " and will result in adding resources and still not being able to turn off safe mode.", e); + LOG.warn("The value of " + DFSConfigKeys.DFS_NAMENODE_CHECKED_VOLUMES_MINIMUM_KEY + + " is greater than the total number of existing storage volumes" + + " and will result in adding resources and still not being able to turn off safe mode.", e); } return NameNodeResourcePolicy.areResourcesAvailable(volumes.values(), minimumRedundantVolumes);