HBASE-26975 Add on heap and off heap memstore info in rs web UI (#4368)

Signed-off-by: Duo Zhang <[email protected]>
(cherry picked from commit cdf81ea)

Author: liangxs

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -101,6 +101,16 @@ public interface MetricsRegionServerWrapper {
    */
   long getMemStoreSize();
 
+  /**
+   * Get the size of the on heap memstore on this region server.
+   */
+  long getOnHeapMemStoreSize();
+
+  /**
+   * Get the size of the off heap memstore on this region server.
+   */
+  long getOffHeapMemStoreSize();
+
   /**
    * Get the total size of the store files this region server is serving from.
    */
@@ -226,7 +236,22 @@ public interface MetricsRegionServerWrapper {
    */
   int getFlushQueueSize();
 
+  /**
+   * Get the limit size of the off heap memstore (if enabled), otherwise
+   * get the limit size of the on heap memstore.
+   */
   long getMemStoreLimit();
+
+  /**
+   * Get the limit size of the on heap memstore.
+   */
+  long getOnHeapMemStoreLimit();
+
+  /**
+   * Get the limit size of the off heap memstore.
+   */
+  long getOffHeapMemStoreLimit();
+
   /**
    * Get the size (in bytes) of the block cache that is free.
    */

hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon

@@ -117,8 +117,9 @@ MetricsRegionServerWrapper mWrap;
         <th>Max Heap</th>
         <th>Direct Memory Used</th>
         <th>Direct Memory Configured</th>
-        <th>Memstore Size</th>
-        <th>Memstore Limit</th>
+        <th>Memstore On-Heap Size / Limit</th>
+        <th>Memstore Off-Heap Size / Limit</th>
+        <th>Memstore Data Size (On&&Off Heap)</th>
     </tr>
 </tr>
 <tr>
@@ -135,10 +136,15 @@ MetricsRegionServerWrapper mWrap;
         <% TraditionalBinaryPrefix.long2String(DirectMemoryUtils.getDirectMemorySize(), "B", 1) %>
     </td>
     <td>
-        <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreSize(), "B", 1) %>
+        <% TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreSize(), "B", 1) + " / "
+         + TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreLimit(), "B", 1) %>
+    </td>
+    <td>
+        <% TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreSize(), "B", 1) + " / "
+         + TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreLimit(), "B", 1) %>
     </td>
     <td>
-        <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreLimit(), "B", 1) %>
+        <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreSize(), "B", 1) %>
     </td>
 </tr>
 </table>

hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java

@@ -30,7 +30,7 @@
 import org.apache.hadoop.hbase.util.Pair;
 
 /**
- * Util class to calculate memory size for memstore, block cache(L1, L2) of RS.
+ * Util class to calculate memory size for memstore(on heap, off heap), block cache(L1, L2) of RS.
  */
 @InterfaceAudience.Private
 public class MemorySizeUtil {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -78,6 +78,8 @@ class MetricsRegionServerWrapperImpl
   private volatile long walFileSize = 0;
   private volatile long numStoreFiles = 0;
   private volatile long memstoreSize = 0;
+  private volatile long onHeapMemstoreSize = 0;
+  private volatile long offHeapMemstoreSize = 0;
   private volatile long storeFileSize = 0;
   private volatile long maxStoreFileAge = 0;
   private volatile long minStoreFileAge = 0;
@@ -281,6 +283,16 @@ public long getMemStoreLimit() {
     return this.regionServer.getRegionServerAccounting().getGlobalMemStoreLimit();
   }
 
+  @Override
+  public long getOnHeapMemStoreLimit() {
+    return this.regionServer.getRegionServerAccounting().getGlobalOnHeapMemStoreLimit();
+  }
+
+  @Override
+  public long getOffHeapMemStoreLimit() {
+    return this.regionServer.getRegionServerAccounting().getGlobalOffHeapMemStoreLimit();
+  }
+
   @Override
   public long getBlockCacheSize() {
     return this.blockCache != null ? this.blockCache.getCurrentSize() : 0L;
@@ -449,6 +461,16 @@ public long getMemStoreSize() {
     return memstoreSize;
   }
 
+  @Override
+  public long getOnHeapMemStoreSize() {
+    return onHeapMemstoreSize;
+  }
+
+  @Override
+  public long getOffHeapMemStoreSize() {
+    return offHeapMemstoreSize;
+  }
+
   @Override
   public long getStoreFileSize() {
     return storeFileSize;
@@ -683,7 +705,8 @@ synchronized public void run() {
         HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
             new HDFSBlocksDistribution();
 
-        long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
+        long tempNumStores = 0, tempNumStoreFiles = 0, tempStoreFileSize = 0;
+        long tempMemstoreSize = 0, tempOnHeapMemstoreSize = 0, tempOffHeapMemstoreSize = 0;
         long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
         long avgAgeNumerator = 0, numHFiles = 0;
         long tempMinStoreFileAge = Long.MAX_VALUE;
@@ -762,6 +785,8 @@ synchronized public void run() {
           for (Store store : storeList) {
             tempNumStoreFiles += store.getStorefilesCount();
             tempMemstoreSize += store.getMemStoreSize().getDataSize();
+            tempOnHeapMemstoreSize += store.getMemStoreSize().getHeapSize();
+            tempOffHeapMemstoreSize += store.getMemStoreSize().getOffHeapSize();
             tempStoreFileSize += store.getStorefilesSize();
 
             OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@@ -856,6 +881,8 @@ synchronized public void run() {
         numStores = tempNumStores;
         numStoreFiles = tempNumStoreFiles;
         memstoreSize = tempMemstoreSize;
+        onHeapMemstoreSize = tempOnHeapMemstoreSize;
+        offHeapMemstoreSize = tempOffHeapMemstoreSize;
         storeFileSize = tempStoreFileSize;
         maxStoreFileAge = tempMaxStoreFileAge;
         if (regionCount > 0) {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java

@@ -81,6 +81,14 @@ long getGlobalMemStoreLimit() {
     return this.globalMemStoreLimit;
   }
 
+  long getGlobalOffHeapMemStoreLimit() {
+    if (isOffheap()) {
+      return this.globalMemStoreLimit;
+    } else {
+      return 0;
+    }
+  }
+
   long getGlobalOnHeapMemStoreLimit() {
     return this.globalOnHeapMemstoreLimit;
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -68,6 +68,16 @@ public long getMemStoreSize() {
     return 1025;
   }
 
+  @Override
+  public long getOnHeapMemStoreSize() {
+    return 500;
+  }
+
+  @Override
+  public long getOffHeapMemStoreSize() {
+    return 600;
+  }
+
   @Override
   public long getStoreFileSize() {
     return 1900;
@@ -253,6 +263,16 @@ public long getMemStoreLimit() {
 	  return 419;
   }
 
+  @Override
+  public long getOnHeapMemStoreLimit() {
+    return 311;
+  }
+
+  @Override
+  public long getOffHeapMemStoreLimit() {
+    return 419;
+  }
+
   @Override
   public long getBlockCacheFreeSize() {
     return 413;