From 067a0041644fefd181040dfd350483c5499dc487 Mon Sep 17 00:00:00 2001 From: Fengnan Li Date: Wed, 14 Apr 2021 17:49:34 -0700 Subject: [PATCH 1/5] HDFS-15810 RBF:Use BigInteger for Router capacity metrics --- .../federation/metrics/FederationMBean.java | 8 +++-- .../metrics/NamenodeBeanMetrics.java | 6 ++-- .../server/federation/metrics/RBFMetrics.java | 30 +++++++++++++++---- .../federation/metrics/TestRBFMetrics.java | 20 +++++++++++++ 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java index c06a2e08548ae..a6a3a551719db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.federation.metrics; +import java.math.BigInteger; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -56,19 +58,19 @@ public interface FederationMBean { * Get the total capacity of the federated cluster. * @return Total capacity of the federated cluster. */ - long getTotalCapacity(); + BigInteger getTotalCapacity(); /** * Get the used capacity of the federated cluster. * @return Used capacity of the federated cluster. */ - long getUsedCapacity(); + BigInteger getUsedCapacity(); /** * Get the remaining capacity of the federated cluster. * @return Remaining capacity of the federated cluster. */ - long getRemainingCapacity(); + BigInteger getRemainingCapacity(); /** * Get the total remote storage capacity mounted in the federated cluster. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java index 0e6e12a0d3e14..e7ef5acf1a26d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java @@ -193,7 +193,7 @@ public String getSoftwareVersion() { @Override public long getUsed() { try { - return getRBFMetrics().getUsedCapacity(); + return getRBFMetrics().getUsedCapacity().longValue(); } catch (IOException e) { LOG.debug("Failed to get the used capacity", e.getMessage()); } @@ -203,7 +203,7 @@ public long getUsed() { @Override public long getFree() { try { - return getRBFMetrics().getRemainingCapacity(); + return getRBFMetrics().getRemainingCapacity().longValue(); } catch (IOException e) { LOG.debug("Failed to get remaining capacity", e.getMessage()); } @@ -213,7 +213,7 @@ public long getFree() { @Override public long getTotal() { try { - return getRBFMetrics().getTotalCapacity(); + return getRBFMetrics().getTotalCapacity().longValue(); } catch (IOException e) { LOG.debug("Failed to Get total capacity", e.getMessage()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java index cc5bf07408246..c0c7dd92614b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.lang.reflect.Method; +import java.math.BigInteger; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -371,13 +372,13 @@ private static void setStateStoreVersions( } @Override - public long getTotalCapacity() { - return getNameserviceAggregatedLong(MembershipStats::getTotalSpace); + public BigInteger getTotalCapacity() { + return getNameserviceAggregatedBigInt(MembershipStats::getTotalSpace); } @Override - public long getRemainingCapacity() { - return getNameserviceAggregatedLong(MembershipStats::getAvailableSpace); + public BigInteger getRemainingCapacity() { + return getNameserviceAggregatedBigInt(MembershipStats::getAvailableSpace); } @Override @@ -386,8 +387,8 @@ public long getProvidedSpace() { } @Override - public long getUsedCapacity() { - return getTotalCapacity() - getRemainingCapacity(); + public BigInteger getUsedCapacity() { + return getTotalCapacity().subtract(getRemainingCapacity()); } @Override @@ -783,6 +784,23 @@ private long getNameserviceAggregatedLong(ToLongFunction f) { } } + private BigInteger getNameserviceAggregatedBigInt( + ToLongFunction f) { + try { + List states = getActiveNamenodeRegistrations(); + BigInteger sum = new BigInteger("0"); + for (MembershipState state : states) { + long lvalue = f.applyAsLong(state.getStats()); + BigInteger bvalue = new BigInteger(String.valueOf(lvalue)); + sum.add(bvalue); + } + return sum; + } catch (IOException e) { + LOG.error("Unable to extract metrics: {}", e.getMessage()); + return new BigInteger("0"); + } + } + /** * Fetches the most active namenode memberships for all known nameservices. * The fetched membership may not or may not be active. Excludes expired diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java index 2c7edaa68e195..12cc32ed20d7c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertFalse; import java.io.IOException; +import java.math.BigInteger; import java.util.Iterator; import java.util.List; @@ -58,6 +59,7 @@ public void testClusterStatsJMX() FederationMBean federationBean = getBean(FEDERATION_BEAN, FederationMBean.class); validateClusterStatsFederationBean(federationBean); + testCapacity(federationBean); RouterMBean routerBean = getBean(ROUTER_BEAN, RouterMBean.class); validateClusterStatsRouterBean(routerBean); } @@ -348,4 +350,22 @@ private void validateClusterStatsRouterBean(RouterMBean bean) { assertTrue(bean.getHostAndPort().length() > 0); assertFalse(bean.isSecurityEnabled()); } + + private void testCapacity(FederationMBean bean) { + List memberships = getActiveMemberships(); + assertTrue(memberships.size() > 1); + + BigInteger availableCapacity = new BigInteger("0"); + BigInteger totalCapacity = new BigInteger("0"); + BigInteger unitCapacity = new BigInteger(String.valueOf(Long.MAX_VALUE)); + for (MembershipState mock : memberships) { + MembershipStats stats = mock.getStats(); + stats.setTotalSpace(Long.MAX_VALUE); + stats.setAvailableSpace(Long.MAX_VALUE); + totalCapacity.add(unitCapacity); + availableCapacity.add(unitCapacity); + } + assertEquals(bean.getTotalCapacity(), totalCapacity); // overflow + assertEquals(bean.getRemainingCapacity(), availableCapacity); // overflow + } } From 3a4e2e57be80e223d9f14b1fcfcebdb6da47ab1b Mon Sep 17 00:00:00 2001 From: Fengnan Li Date: Sun, 18 Apr 2021 22:55:19 -0700 Subject: [PATCH 2/5] Add new metrics in bigint instead of changing existing one --- .../src/site/markdown/Metrics.md | 9 +++-- .../federation/metrics/FederationMBean.java | 30 +++++++++++++++-- .../metrics/NamenodeBeanMetrics.java | 6 ++-- .../server/federation/metrics/RBFMetrics.java | 28 ++++++++++++---- .../federation/metrics/TestMetricsBase.java | 11 +++++++ .../federation/metrics/TestRBFMetrics.java | 33 ++++++++++++------- 6 files changed, 89 insertions(+), 28 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 8423b5351b11f..6cec030ee2557 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -527,9 +527,12 @@ RBFMetrics shows the metrics which are the aggregated values of sub-clusters' in | `NumInMaintenanceLiveDataNodes` | Number of live Datanodes which are in maintenance state | | `NumInMaintenanceDeadDataNodes` | Number of dead Datanodes which are in maintenance state | | `NumEnteringMaintenanceDataNodes` | Number of Datanodes that are entering the maintenance state | -| `TotalCapacity` | Current raw capacity of DataNodes in bytes | -| `UsedCapacity` | Current used capacity across all DataNodes in bytes | -| `RemainingCapacity` | Current remaining capacity in bytes | +| `TotalCapacity` | Current raw capacity of DataNodes in bytes (long primitive, may overflow) | +| `UsedCapacity` | Current used capacity across all DataNodes in bytes (long primitive, may overflow) | +| `RemainingCapacity` | Current remaining capacity in bytes (long primitive, may overflow) | +| `TotalCapacityBigInt` | Current raw capacity of DataNodes in bytes (using BigInteger) | +| `UsedCapacityBigInt` | Current used capacity across all DataNodes in bytes (using BigInteger) | +| `RemainingCapacityBigInt` | Current remaining capacity in bytes (using BigInteger) | | `NumOfMissingBlocks` | Current number of missing blocks | | `NumLiveNodes` | Number of datanodes which are currently live | | `NumDeadNodes` | Number of datanodes which are currently dead | diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java index a6a3a551719db..b9ea8709e90f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java @@ -56,21 +56,45 @@ public interface FederationMBean { /** * Get the total capacity of the federated cluster. + * The number could overflow if too big. In that case use + * {@link #getTotalCapacityBigInt()} instead. * @return Total capacity of the federated cluster. */ - BigInteger getTotalCapacity(); + long getTotalCapacity(); /** * Get the used capacity of the federated cluster. + * The number could overflow if too big. In that case use + * {@link #getUsedCapacityBigInt()} instead. * @return Used capacity of the federated cluster. */ - BigInteger getUsedCapacity(); + long getUsedCapacity(); /** * Get the remaining capacity of the federated cluster. + * The number could overflow if too big. In that case use + * {@link #getRemainingCapacityBigInt()} instead. * @return Remaining capacity of the federated cluster. */ - BigInteger getRemainingCapacity(); + long getRemainingCapacity(); + + /** + * Get the total capacity (big integer) of the federated cluster. + * @return Total capacity of the federated cluster. + */ + BigInteger getTotalCapacityBigInt(); + + /** + * Get the used capacity (big integer) of the federated cluster. + * @return Used capacity of the federated cluster. + */ + BigInteger getUsedCapacityBigInt(); + + /** + * Get the remaining capacity (big integer) of the federated cluster. + * @return Remaining capacity of the federated cluster. + */ + BigInteger getRemainingCapacityBigInt(); /** * Get the total remote storage capacity mounted in the federated cluster. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java index e7ef5acf1a26d..0e6e12a0d3e14 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java @@ -193,7 +193,7 @@ public String getSoftwareVersion() { @Override public long getUsed() { try { - return getRBFMetrics().getUsedCapacity().longValue(); + return getRBFMetrics().getUsedCapacity(); } catch (IOException e) { LOG.debug("Failed to get the used capacity", e.getMessage()); } @@ -203,7 +203,7 @@ public long getUsed() { @Override public long getFree() { try { - return getRBFMetrics().getRemainingCapacity().longValue(); + return getRBFMetrics().getRemainingCapacity(); } catch (IOException e) { LOG.debug("Failed to get remaining capacity", e.getMessage()); } @@ -213,7 +213,7 @@ public long getFree() { @Override public long getTotal() { try { - return getRBFMetrics().getTotalCapacity().longValue(); + return getRBFMetrics().getTotalCapacity(); } catch (IOException e) { LOG.debug("Failed to Get total capacity", e.getMessage()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java index c0c7dd92614b1..ec41a32716607 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java @@ -372,12 +372,27 @@ private static void setStateStoreVersions( } @Override - public BigInteger getTotalCapacity() { + public long getTotalCapacity() { + return getNameserviceAggregatedLong(MembershipStats::getTotalSpace); + } + + @Override + public long getRemainingCapacity() { + return getNameserviceAggregatedLong(MembershipStats::getAvailableSpace); + } + + @Override + public long getUsedCapacity() { + return getTotalCapacity() - getRemainingCapacity(); + } + + @Override + public BigInteger getTotalCapacityBigInt() { return getNameserviceAggregatedBigInt(MembershipStats::getTotalSpace); } @Override - public BigInteger getRemainingCapacity() { + public BigInteger getRemainingCapacityBigInt() { return getNameserviceAggregatedBigInt(MembershipStats::getAvailableSpace); } @@ -387,8 +402,8 @@ public long getProvidedSpace() { } @Override - public BigInteger getUsedCapacity() { - return getTotalCapacity().subtract(getRemainingCapacity()); + public BigInteger getUsedCapacityBigInt() { + return getTotalCapacityBigInt().subtract(getRemainingCapacityBigInt()); } @Override @@ -788,11 +803,10 @@ private BigInteger getNameserviceAggregatedBigInt( ToLongFunction f) { try { List states = getActiveNamenodeRegistrations(); - BigInteger sum = new BigInteger("0"); + BigInteger sum = BigInteger.valueOf(0); for (MembershipState state : states) { long lvalue = f.applyAsLong(state.getStats()); - BigInteger bvalue = new BigInteger(String.valueOf(lvalue)); - sum.add(bvalue); + sum = sum.add(BigInteger.valueOf(lvalue)); } return sum; } catch (IOException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java index 4759d05f820dc..b01e22006d776 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestMetricsBase.java @@ -259,4 +259,15 @@ private MembershipState createRegistration(String ns, String nn, assertTrue(response.getResult()); return record; } + + // refresh namenode registration for new attributes + public boolean refreshNamenodeRegistration(NamenodeHeartbeatRequest request) + throws IOException { + boolean result = membershipStore.namenodeHeartbeat(request).getResult(); + membershipStore.loadCache(true); + MembershipNamenodeResolver resolver = + (MembershipNamenodeResolver) router.getNamenodeResolver(); + resolver.loadCache(true); + return result; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java index 12cc32ed20d7c..3dd7776cf63e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java @@ -18,10 +18,7 @@ package org.apache.hadoop.hdfs.server.federation.metrics; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getBean; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.*; import java.io.IOException; import java.math.BigInteger; @@ -32,6 +29,7 @@ import org.apache.commons.collections.ListUtils; import org.apache.hadoop.hdfs.server.federation.router.Router; +import org.apache.hadoop.hdfs.server.federation.store.protocol.NamenodeHeartbeatRequest; import org.apache.hadoop.hdfs.server.federation.store.records.MembershipState; import org.apache.hadoop.hdfs.server.federation.store.records.MembershipStats; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; @@ -351,21 +349,32 @@ private void validateClusterStatsRouterBean(RouterMBean bean) { assertFalse(bean.isSecurityEnabled()); } - private void testCapacity(FederationMBean bean) { + private void testCapacity(FederationMBean bean) throws IOException { List memberships = getActiveMemberships(); assertTrue(memberships.size() > 1); - BigInteger availableCapacity = new BigInteger("0"); - BigInteger totalCapacity = new BigInteger("0"); - BigInteger unitCapacity = new BigInteger(String.valueOf(Long.MAX_VALUE)); + BigInteger availableCapacity = BigInteger.valueOf(0); + BigInteger totalCapacity = BigInteger.valueOf(0); + BigInteger unitCapacity = BigInteger.valueOf(Long.MAX_VALUE); for (MembershipState mock : memberships) { MembershipStats stats = mock.getStats(); stats.setTotalSpace(Long.MAX_VALUE); stats.setAvailableSpace(Long.MAX_VALUE); - totalCapacity.add(unitCapacity); - availableCapacity.add(unitCapacity); + // reset stats to make the new value persistent + mock.setStats(stats); + // write back the new namenode information to state store + refreshNamenodeRegistration(NamenodeHeartbeatRequest.newInstance(mock)); + totalCapacity = totalCapacity.add(unitCapacity); + availableCapacity = availableCapacity.add(unitCapacity); } - assertEquals(bean.getTotalCapacity(), totalCapacity); // overflow - assertEquals(bean.getRemainingCapacity(), availableCapacity); // overflow + + // for local cache update + assertEquals(bean.getTotalCapacityBigInt(), totalCapacity); + // not equal since overflow happened. + assertNotEquals(BigInteger.valueOf(bean.getTotalCapacity()), totalCapacity); + assertEquals(bean.getRemainingCapacityBigInt(), availableCapacity); + // not equal since overflow happened. + assertNotEquals( + BigInteger.valueOf(bean.getRemainingCapacity()), availableCapacity); } } From c76af4e0d5ee89a620f8a94a28f7be4666675159 Mon Sep 17 00:00:00 2001 From: Fengnan Li Date: Fri, 23 Apr 2021 10:44:24 -0700 Subject: [PATCH 3/5] Comments --- .../src/main/webapps/router/federationhealth.html | 6 +++--- .../hdfs/server/federation/metrics/TestRBFMetrics.java | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html index 87d0e718bdcd3..eca395ff4b2a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html @@ -113,9 +113,9 @@

Non Heap Memory used {used|fmt_bytes} of {committed|fmt_bytes} Commited Non Heap Memory. Max Non Heap Memory is {@eq key=max value="-1" type="number"}<unbounded>{:else}{max|fmt_bytes}{/eq}.

{/mem.NonHeapMemoryUsage} - - - + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java index 3dd7776cf63e6..80889e0e60cf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java @@ -18,7 +18,11 @@ package org.apache.hadoop.hdfs.server.federation.metrics; import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getBean; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.math.BigInteger; @@ -349,7 +353,7 @@ private void validateClusterStatsRouterBean(RouterMBean bean) { assertFalse(bean.isSecurityEnabled()); } - private void testCapacity(FederationMBean bean) throws IOException { + private void testCapacity(FederationMBean bean) throws IOException { List memberships = getActiveMemberships(); assertTrue(memberships.size() > 1); From 92264cd03a6cc254a305009ac3fbd99e45c61715 Mon Sep 17 00:00:00 2001 From: Fengnan Li Date: Tue, 27 Apr 2021 09:39:44 -0700 Subject: [PATCH 4/5] Comments --- .../hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java index 80889e0e60cf6..9d9cf13a5e3af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java @@ -367,7 +367,8 @@ private void testCapacity(FederationMBean bean) throws IOException { // reset stats to make the new value persistent mock.setStats(stats); // write back the new namenode information to state store - refreshNamenodeRegistration(NamenodeHeartbeatRequest.newInstance(mock)); + assertTrue(refreshNamenodeRegistration( + NamenodeHeartbeatRequest.newInstance(mock))); totalCapacity = totalCapacity.add(unitCapacity); availableCapacity = availableCapacity.add(unitCapacity); } From 5e2857499a97d80f4ea02b942cb3b856592fe11d Mon Sep 17 00:00:00 2001 From: Fengnan Li Date: Sat, 1 May 2021 23:01:31 -0700 Subject: [PATCH 5/5] Comments for tests --- .../hdfs/server/federation/metrics/TestRBFMetrics.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java index 9d9cf13a5e3af..25473f8df9233 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRBFMetrics.java @@ -374,12 +374,12 @@ private void testCapacity(FederationMBean bean) throws IOException { } // for local cache update - assertEquals(bean.getTotalCapacityBigInt(), totalCapacity); + assertEquals(totalCapacity, bean.getTotalCapacityBigInt()); // not equal since overflow happened. - assertNotEquals(BigInteger.valueOf(bean.getTotalCapacity()), totalCapacity); - assertEquals(bean.getRemainingCapacityBigInt(), availableCapacity); + assertNotEquals(totalCapacity, BigInteger.valueOf(bean.getTotalCapacity())); + assertEquals(availableCapacity, bean.getRemainingCapacityBigInt()); // not equal since overflow happened. - assertNotEquals( - BigInteger.valueOf(bean.getRemainingCapacity()), availableCapacity); + assertNotEquals(availableCapacity, + BigInteger.valueOf(bean.getRemainingCapacity())); } }
Total capacity{TotalCapacity|fmt_bytes}
Used capacity{UsedCapacity|fmt_bytes}
Remaining capacity{RemainingCapacity|fmt_bytes}
Total capacity{TotalCapacityBigInt|fmt_bytes}
Used capacity{UsedCapacityBigInt|fmt_bytes}
Remaining capacity{RemainingCapacityBigInt|fmt_bytes}
Nameservices{NumNameservices}
Namenodes{NumNamenodes}