HBASE-26154: Adds exception metrics for QuotaExceededException and RpcThrottlingException (#3546)

bbeaudreault · web-flow · commit 63b17d16745c · 2021-08-02T15:19:59.000+05:30
Signed-off-by: Xiaolin Ha &lt;haxiaolin@apache.org&gt;
Signed-off-by: Pankaj Kumar&lt;pankajkumar@apache.org&gt;
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSource.java
@@ -41,6 +41,8 @@ public interface ExceptionTrackingSource extends BaseSource {
       "rest of the requests will have to be retried.";
   String EXCEPTIONS_CALL_QUEUE_TOO_BIG = "exceptions.callQueueTooBig";
   String EXCEPTIONS_CALL_QUEUE_TOO_BIG_DESC = "Call queue is full";
+  String EXCEPTIONS_QUOTA_EXCEEDED = "exceptions.quotaExceeded";
+  String EXCEPTIONS_RPC_THROTTLING = "exceptions.rpcThrottling";
 
   void exception();
 
@@ -56,4 +58,6 @@ public interface ExceptionTrackingSource extends BaseSource {
   void tooBusyException();
   void multiActionTooLargeException();
   void callQueueTooBigException();
+  void quotaExceededException();
+  void rpcThrottlingException();
 }
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/ExceptionTrackingSourceImpl.java
@@ -38,6 +38,8 @@ public class ExceptionTrackingSourceImpl extends BaseSourceImpl
   protected MutableFastCounter exceptionsMoved;
   protected MutableFastCounter exceptionsMultiTooLarge;
   protected MutableFastCounter exceptionsCallQueueTooBig;
+  protected MutableFastCounter exceptionsQuotaExceeded;
+  protected MutableFastCounter exceptionsRpcThrottling;
 
   public ExceptionTrackingSourceImpl(String metricsName, String metricsDescription,
                                      String metricsContext, String metricsJmxContext) {
@@ -66,6 +68,10 @@ public void init() {
         .newCounter(EXCEPTIONS_MULTI_TOO_LARGE_NAME, EXCEPTIONS_MULTI_TOO_LARGE_DESC, 0L);
     this.exceptionsCallQueueTooBig = this.getMetricsRegistry()
         .newCounter(EXCEPTIONS_CALL_QUEUE_TOO_BIG, EXCEPTIONS_CALL_QUEUE_TOO_BIG_DESC, 0L);
+    this.exceptionsQuotaExceeded = this.getMetricsRegistry()
+      .newCounter(EXCEPTIONS_QUOTA_EXCEEDED, EXCEPTIONS_TYPE_DESC, 0L);
+    this.exceptionsRpcThrottling = this.getMetricsRegistry()
+      .newCounter(EXCEPTIONS_RPC_THROTTLING, EXCEPTIONS_TYPE_DESC, 0L);
   }
 
   @Override
@@ -117,4 +123,14 @@ public void multiActionTooLargeException() {
   public void callQueueTooBigException() {
     exceptionsCallQueueTooBig.incr();
   }
+
+  @Override
+  public void quotaExceededException() {
+    exceptionsQuotaExceeded.incr();
+  }
+
+  @Override
+  public void rpcThrottlingException() {
+    exceptionsRpcThrottling.incr();
+  }
 }
diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java
@@ -184,7 +184,7 @@ public void assertGaugeLt(String name, double expected, BaseSource source) {
   @Override
   public void assertCounter(String name, long expected, BaseSource source) {
     long found = getCounter(name, source);
-    assertEquals("Metrics Counters should be equal", (long) Long.valueOf(expected), found);
+    assertEquals(name + "(" + found + ") should be equal", (long) Long.valueOf(expected), found);
   }
 
   @Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServer.java
@@ -24,15 +24,21 @@
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.RegionTooBusyException;
 import org.apache.hadoop.hbase.UnknownScannerException;
+import org.apache.hadoop.hbase.quotas.QuotaExceededException;
+import org.apache.hadoop.hbase.quotas.RpcThrottlingException;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
 import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
 import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
 import org.apache.hadoop.hbase.exceptions.ScannerResetException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @InterfaceAudience.Private
 public class MetricsHBaseServer {
+  private static final Logger LOG = LoggerFactory.getLogger(MetricsHBaseServer.class);
+
   private MetricsHBaseServerSource source;
   private MetricsHBaseServerWrapper serverWrapper;
 
@@ -116,6 +122,12 @@ public void exception(Throwable throwable) {
         source.multiActionTooLargeException();
       } else if (throwable instanceof CallQueueTooBigException) {
         source.callQueueTooBigException();
+      } else if (throwable instanceof QuotaExceededException) {
+        source.quotaExceededException();
+      } else if (throwable instanceof RpcThrottlingException) {
+        source.rpcThrottlingException();
+      } else if (LOG.isDebugEnabled()) {
+        LOG.debug("Unknown exception type", throwable);
       }
     }
   }
diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftMetrics.java
@@ -31,9 +31,13 @@
 import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
 import org.apache.hadoop.hbase.exceptions.ScannerResetException;
+import org.apache.hadoop.hbase.quotas.QuotaExceededException;
+import org.apache.hadoop.hbase.quotas.RpcThrottlingException;
 import org.apache.hadoop.hbase.thrift.generated.IOError;
 import org.apache.hadoop.hbase.thrift2.generated.TIOError;
 import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class is for maintaining the various statistics of thrift server
@@ -42,6 +46,7 @@
 @InterfaceAudience.Private
 public class ThriftMetrics  {
 
+  private static final Logger LOG = LoggerFactory.getLogger(ThriftMetrics.class);
 
   public enum ThriftServerType {
     ONE,
@@ -143,6 +148,12 @@ public void exception(Throwable rawThrowable) {
         source.multiActionTooLargeException();
       } else if (throwable instanceof CallQueueTooBigException) {
         source.callQueueTooBigException();
+      } else if (throwable instanceof QuotaExceededException) {
+        source.quotaExceededException();
+      } else if (throwable instanceof RpcThrottlingException) {
+        source.rpcThrottlingException();
+      } else if (LOG.isDebugEnabled()) {
+        LOG.debug("Unknown exception type", throwable);
       }
     }
   }
diff --git a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift/ErrorThrowingGetObserver.java b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift/ErrorThrowingGetObserver.java
@@ -40,6 +40,8 @@
 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
 import org.apache.hadoop.hbase.exceptions.ScannerResetException;
 import org.apache.hadoop.hbase.metrics.ExceptionTrackingSource;
+import org.apache.hadoop.hbase.quotas.QuotaExceededException;
+import org.apache.hadoop.hbase.quotas.RpcThrottlingException;
 import org.apache.hadoop.hbase.util.Bytes;
 
 /**
@@ -79,6 +81,10 @@ public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e,
           throw new RegionTooBusyException("Failing for test");
         case OUT_OF_ORDER_SCANNER_NEXT:
           throw new OutOfOrderScannerNextException("Failing for test");
+        case QUOTA_EXCEEDED:
+          throw new QuotaExceededException("Failing for test");
+        case RPC_THROTTLING:
+          throw new RpcThrottlingException("Failing for test");
         default:
           throw new DoNotRetryIOException("Failing for test");
       }
@@ -94,7 +100,9 @@ public enum ErrorType {
     SCANNER_RESET(ExceptionTrackingSource.EXCEPTIONS_SCANNER_RESET_NAME),
     UNKNOWN_SCANNER(ExceptionTrackingSource.EXCEPTIONS_UNKNOWN_NAME),
     REGION_TOO_BUSY(ExceptionTrackingSource.EXCEPTIONS_BUSY_NAME),
-    OUT_OF_ORDER_SCANNER_NEXT(ExceptionTrackingSource.EXCEPTIONS_OOO_NAME);
+    OUT_OF_ORDER_SCANNER_NEXT(ExceptionTrackingSource.EXCEPTIONS_OOO_NAME),
+    QUOTA_EXCEEDED(ExceptionTrackingSource.EXCEPTIONS_QUOTA_EXCEEDED),
+    RPC_THROTTLING(ExceptionTrackingSource.EXCEPTIONS_RPC_THROTTLING);
 
     private final String metricName;
 

Original file line number	Diff line number	Diff line change
`@@ -184,7 +184,7 @@ public void assertGaugeLt(String name, double expected, BaseSource source) {`
`184`	`184`	`@Override`
`185`	`185`	`public void assertCounter(String name, long expected, BaseSource source) {`
`186`	`186`	`long found = getCounter(name, source);`
`187`		`- assertEquals("Metrics Counters should be equal", (long) Long.valueOf(expected), found);`
	`187`	`+ assertEquals(name + "(" + found + ") should be equal", (long) Long.valueOf(expected), found);`
`188`	`188`	`}`
`189`	`189`
`190`	`190`	`@Override`