Separate RowCache from BlockCache

- Implement RowCache
  - Initially considered modifying LruBlockCache, but the required changes were extensive.
    Instead, implemented RowCache using Caffeine cache.
- Add row.cache.size configuration
  - Default is 0.0 (disabled); RowCache is enabled only if explicitly set to a value > 0.
  - The combined size of BlockCache + MemStore + RowCache must not exceed 80% of the heap.
- Add Row Cache tab to RegionServer Block Cache UI
  - RowCache is not a BlockCache, but added here since there is no better place.
- Add RowCache metrics
  - Metrics for size, count, eviction, hit, and miss are now exposed.

Author: terence-yoo

hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java

@@ -234,7 +234,7 @@ public class TableDescriptorBuilder {
   @InterfaceAudience.Private
   public static final String ROW_CACHE_ENABLED = "ROW_CACHE_ENABLED";
   private static final Bytes ROW_CACHE_ENABLED_KEY = new Bytes(Bytes.toBytes(ROW_CACHE_ENABLED));
-  private static final boolean DEFAULT_ROW_CACHE_ENABLED = false;
+  private static final boolean DEFAULT_ROW_CACHE_ENABLED = true;
 
   static {
     DEFAULT_VALUES.put(MAX_FILESIZE, String.valueOf(HConstants.DEFAULT_MAX_FILE_SIZE));

hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java

@@ -1017,6 +1017,13 @@ public enum OperationStatusCode {
 
   public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.4f;
 
+  /**
+   * Configuration key for the size of the row cache
+   */
+  public static final String ROW_CACHE_SIZE_KEY = "row.cache.size";
+
+  public static final float ROW_CACHE_SIZE_DEFAULT = 0.0f;
+
   /**
    * Configuration key for the memory size of the block cache
    */

hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockType.java

@@ -80,14 +80,6 @@ public int getId() {
   /** Fixed file trailer, both versions (always just a magic string) */
   TRAILER("TRABLK\"$", BlockCategory.META),
 
-  // Pseudo block
-
-  /**
-   * Cells of a row for row cache. This is a pseudo block type. It only exists to share the
-   * BlockCache interface.
-   */
-  ROW_CELLS("ROWCELLS", BlockCategory.ROW),
-
   // Legacy blocks
 
   /** Block index magic string in version 1 */
@@ -99,7 +91,6 @@ public enum BlockCategory {
     INDEX,
     BLOOM,
     ALL_CATEGORIES,
-    ROW,
     UNKNOWN;
 
     /**

hbase-common/src/main/resources/hbase-default.xml

@@ -1027,6 +1027,12 @@ possible configurations would overwhelm and obscure the important.
       This configuration allows setting an absolute memory size instead of a percentage of the maximum heap.
       Takes precedence over hfile.block.cache.size if both are specified.</description>
   </property>
+  <property>
+    <name>row.cache.size</name>
+    <value>0.0</value>
+    <description>Percentage of maximum heap (-Xmx setting) to allocate to row cache.
+      Default of 0.0; it means the row cache is disabled.</description>
+  </property>
   <property>
       <name>hfile.block.index.cacheonwrite</name>
       <value>false</value>

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -386,7 +386,6 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String BLOCK_CACHE_GENERAL_BLOOM_META_MISS_COUNT = "blockCacheGeneralBloomMetaMissCount";
   String BLOCK_CACHE_DELETE_FAMILY_BLOOM_MISS_COUNT = "blockCacheDeleteFamilyBloomMissCount";
   String BLOCK_CACHE_TRAILER_MISS_COUNT = "blockCacheTrailerMissCount";
-  String BLOCK_CACHE_ROW_MISS_COUNT = "blockCacheRowMissCount";
   String BLOCK_CACHE_DATA_HIT_COUNT = "blockCacheDataHitCount";
   String BLOCK_CACHE_ENCODED_DATA_HIT_COUNT = "blockCacheEncodedDataHitCount";
   String BLOCK_CACHE_LEAF_INDEX_HIT_COUNT = "blockCacheLeafIndexHitCount";
@@ -398,7 +397,6 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String BLOCK_CACHE_GENERAL_BLOOM_META_HIT_COUNT = "blockCacheGeneralBloomMetaHitCount";
   String BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT = "blockCacheDeleteFamilyBloomHitCount";
   String BLOCK_CACHE_TRAILER_HIT_COUNT = "blockCacheTrailerHitCount";
-  String BLOCK_CACHE_ROW_HIT_COUNT = "blockCacheRowHitCount";
   String L1_CACHE_FREE_SIZE = "l1CacheFreeSize";
   String L1_CACHE_FREE_SIZE_DESC = "Amount of free bytes in the L1 cache";
   String L1_CACHE_SIZE = "l1CacheSize";
@@ -432,6 +430,13 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String L2_CACHE_HIT_RATIO_DESC = "L2 cache hit ratio.";
   String L2_CACHE_MISS_RATIO = "l2CacheMissRatio";
   String L2_CACHE_MISS_RATIO_DESC = "L2 cache miss ratio.";
+
+  String ROW_CACHE_HIT_COUNT = "rowCacheHitCount";
+  String ROW_CACHE_MISS_COUNT = "rowCacheMissCount";
+  String ROW_CACHE_EVICTED_ROW_COUNT = "rowCacheEvictedRowCount";
+  String ROW_CACHE_SIZE = "rowCacheSize";
+  String ROW_CACHE_COUNT = "rowCacheCount";
+
   String RS_START_TIME_NAME = "regionServerStartTime";
   String ZOOKEEPER_QUORUM_NAME = "zookeeperQuorum";
   String SERVER_NAME_NAME = "serverName";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java

@@ -436,7 +436,6 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
         .addCounter(Interns.info(BLOCK_CACHE_DELETE_FAMILY_BLOOM_MISS_COUNT, ""),
           rsWrap.getDeleteFamilyBloomMissCount())
         .addCounter(Interns.info(BLOCK_CACHE_TRAILER_MISS_COUNT, ""), rsWrap.getTrailerMissCount())
-        .addCounter(Interns.info(BLOCK_CACHE_ROW_MISS_COUNT, ""), rsWrap.getRowMissCount())
         .addCounter(Interns.info(BLOCK_CACHE_DATA_HIT_COUNT, ""), rsWrap.getDataHitCount())
         .addCounter(Interns.info(BLOCK_CACHE_LEAF_INDEX_HIT_COUNT, ""),
           rsWrap.getLeafIndexHitCount())
@@ -453,7 +452,12 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
         .addCounter(Interns.info(BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT, ""),
           rsWrap.getDeleteFamilyBloomHitCount())
         .addCounter(Interns.info(BLOCK_CACHE_TRAILER_HIT_COUNT, ""), rsWrap.getTrailerHitCount())
-        .addCounter(Interns.info(BLOCK_CACHE_ROW_HIT_COUNT, ""), rsWrap.getRowHitCount())
+        .addCounter(Interns.info(ROW_CACHE_HIT_COUNT, ""), rsWrap.getRowCacheHitCount())
+        .addCounter(Interns.info(ROW_CACHE_MISS_COUNT, ""), rsWrap.getRowCacheMissCount())
+        .addCounter(Interns.info(ROW_CACHE_EVICTED_ROW_COUNT, ""),
+          rsWrap.getRowCacheEvictedRowCount())
+        .addGauge(Interns.info(ROW_CACHE_SIZE, ""), rsWrap.getRowCacheSize())
+        .addGauge(Interns.info(ROW_CACHE_COUNT, ""), rsWrap.getRowCacheCount())
         .addCounter(Interns.info(UPDATES_BLOCKED_TIME, UPDATES_BLOCKED_DESC),
           rsWrap.getUpdatesBlockedTime())
         .addCounter(Interns.info(FLUSHED_CELLS, FLUSHED_CELLS_DESC), rsWrap.getFlushedCellsCount())

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -615,8 +615,6 @@ public interface MetricsRegionServerWrapper {
 
   long getTrailerMissCount();
 
-  long getRowMissCount();
-
   long getDataHitCount();
 
   long getLeafIndexHitCount();
@@ -637,7 +635,15 @@ public interface MetricsRegionServerWrapper {
 
   long getTrailerHitCount();
 
-  long getRowHitCount();
+  long getRowCacheHitCount();
+
+  long getRowCacheMissCount();
+
+  long getRowCacheSize();
+
+  long getRowCacheCount();
+
+  long getRowCacheEvictedRowCount();
 
   long getTotalRowActionRequestCount();
 

hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon

@@ -21,6 +21,7 @@ Template for rendering Block Cache tabs in RegionServer Status page.
 CacheConfig cacheConfig;
 Configuration config;
 BlockCache bc;
+RowCache rowCache;
 </%args>
 <%java>
   String bcUrl = bc == null ? null : "http://hbase.apache.org/devapidocs/" + bc.getClass().getName().replaceAll("\\.", "/") + ".html";
@@ -42,6 +43,7 @@ org.apache.hadoop.hbase.io.hfile.bucket.BucketCacheStats;
 org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
 org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator;
 org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.Bucket;
+org.apache.hadoop.hbase.regionserver.RowCache;
 org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
 </%import>
 <div class="tabbable">
@@ -51,6 +53,7 @@ org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
         <li class="nav-item"><a class="nav-link" href="#tab_bc_stats" data-bs-toggle="tab" role="tab">Stats</a></li>
         <li class="nav-item"><a class="nav-link" href="#tab_bc_l1" data-bs-toggle="tab" role="tab">L1</a></li>
         <li class="nav-item"><a class="nav-link" href="#tab_bc_l2" data-bs-toggle="tab" role="tab">L2</a></li>
+        <li class="nav-item"><a class="nav-link" href="#tab_row_cache" data-bs-toggle="tab" role="tab">Row Cache</a></li>
     </ul>
     <div class="tab-content">
         <div class="tab-pane active" id="tab_bc_baseInfo" role="tabpanel">
@@ -68,6 +71,9 @@ org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
         <div class="tab-pane" id="tab_bc_l2" role="tabpanel">
             <& bc_l; bc = bcs == null? null: bcs.length <= 1? null: bcs[1]; name = "L2"; evictions = evictions; &>
         </div>
+        <div class="tab-pane" id="tab_row_cache" role="tabpanel">
+            <& row_cache_stats; rowCache = rowCache &>
+        </div>
     </div>
 </div>
 
@@ -562,3 +568,55 @@ Increase that value to get a complete picture.
 </p>
 </%def>
 
+<%def row_cache_stats>
+<%args>
+    RowCache rowCache;
+</%args>
+<%if rowCache == null %>
+<p>RowCache is null</p>
+<%else>
+<table class="table table-striped">
+    <tr>
+        <th>Attribute</th>
+        <th>Value</th>
+        <th>Description</th>
+    </tr>
+    <tr>
+        <td>Size</td>
+        <td><% TraditionalBinaryPrefix.long2String(rowCache.getSize(), "B", 1) %></td>
+        <td>Current size of row cache in use</td>
+    </tr>
+    <tr>
+        <td>Free</td>
+        <td><% TraditionalBinaryPrefix.long2String(rowCache.getMaxSize() - rowCache.getSize(), "B", 1) %></td>
+        <td>The total free memory currently available to store more cache entries</td>
+    </tr>
+    <tr>
+        <td>Count</td>
+        <td><% String.format("%,d", rowCache.getCount()) %></td>
+        <td>The number of rows in row cache</td>
+    </tr>
+    <tr>
+        <td>Evicted Rows</td>
+        <td><% String.format("%,d", rowCache.getEvictedRowCount()) %></td>
+        <td>The total number of rows evicted</td>
+    </tr>
+    <tr>
+        <td>Hits</td>
+        <td><% String.format("%,d", rowCache.getHitCount()) %></td>
+        <td>The number requests that were cache hits</td>
+    </tr>
+    <tr>
+        <td>Misses</td>
+        <td><% String.format("%,d", rowCache.getMissCount()) %></td>
+        <td>The number requests that were cache misses</td>
+    </tr>
+    <tr>
+        <td>All Time Hit Ratio</td>
+        <td><% String.format("%,.2f", rowCache.getHitCount() * 100.0 / (rowCache.getMissCount() + rowCache.getHitCount())) %><% "%" %></td>
+        <td>Hit Count divided by total requests count</td>
+    </tr>
+</table>
+<p>RowCache is a separate cache distinct from BlockCache. Since there is no more appropriate place to display RowCache information, it is shown here alongside BlockCache.</p>
+</%if>
+</%def>

hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon

@@ -26,10 +26,12 @@ String bcv = "";
 <%import>
 java.util.*;
 org.apache.hadoop.hbase.regionserver.HRegionServer;
+org.apache.hadoop.hbase.regionserver.RowCacheService;
 org.apache.hadoop.hbase.client.RegionInfo;
 org.apache.hadoop.hbase.ServerName;
 org.apache.hadoop.hbase.HBaseConfiguration;
 org.apache.hadoop.hbase.io.hfile.CacheConfig;
+org.apache.hadoop.hbase.regionserver.RowCache;
 org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ServerInfo;
 org.apache.hadoop.hbase.util.JvmVersion;
@@ -51,6 +53,8 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
   ServerName masterServerName = masterAddressTracker == null ? null
     : masterAddressTracker.getMasterAddress();
   int infoPort = masterAddressTracker == null ? 0 : masterAddressTracker.getMasterInfoPort();
+  RowCacheService rowCacheService = regionServer.getRSRpcServices().getRowCacheService();
+  RowCache rowCache = rowCacheService == null ? null : rowCacheService.getRowCache();
 </%java>
 
 <%class>
@@ -149,7 +153,7 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
 
     <section>
     <h2>Block Cache</h2>
-    <& BlockCacheTmpl; cacheConfig = new CacheConfig(regionServer.getConfiguration()); config = regionServer.getConfiguration(); bc = regionServer.getBlockCache().orElse(null) &>
+    <& BlockCacheTmpl; cacheConfig = new CacheConfig(regionServer.getConfiguration()); config = regionServer.getConfiguration(); bc = regionServer.getBlockCache().orElse(null); rowCache = rowCache &>
     </section>
 
     <section>

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java

@@ -91,7 +91,6 @@ public class CacheStats {
   private final LongAdder generalBloomMetaMissCount = new LongAdder();
   private final LongAdder deleteFamilyBloomMissCount = new LongAdder();
   private final LongAdder trailerMissCount = new LongAdder();
-  private final LongAdder rowMissCount = new LongAdder();
 
   private final LongAdder dataHitCount = new LongAdder();
   private final LongAdder leafIndexHitCount = new LongAdder();
@@ -103,7 +102,6 @@ public class CacheStats {
   private final LongAdder generalBloomMetaHitCount = new LongAdder();
   private final LongAdder deleteFamilyBloomHitCount = new LongAdder();
   private final LongAdder trailerHitCount = new LongAdder();
-  private final LongAdder rowHitCount = new LongAdder();
 
   // Executor for periodic cache stats rolling
   private ScheduledExecutorService metricsRollerScheduler;
@@ -221,9 +219,6 @@ public void miss(boolean caching, boolean primary, BlockType type) {
       case TRAILER:
         trailerMissCount.increment();
         break;
-      case ROW_CELLS:
-        rowMissCount.increment();
-        break;
       default:
         // If there's a new type that's fine
         // Ignore it for now. This is metrics don't exception.
@@ -271,9 +266,6 @@ public void hit(boolean caching, boolean primary, BlockType type) {
       case TRAILER:
         trailerHitCount.increment();
         break;
-      case ROW_CELLS:
-        rowHitCount.increment();
-        break;
       default:
         // If there's a new type that's fine
         // Ignore it for now. This is metrics don't exception.
@@ -384,14 +376,6 @@ public long getTrailerHitCount() {
     return trailerHitCount.sum();
   }
 
-  public long getRowHitCount() {
-    return rowHitCount.sum();
-  }
-
-  public long getRowMissCount() {
-    return rowMissCount.sum();
-  }
-
   public long getRequestCount() {
     return getHitCount() + getMissCount();
   }