HBASE-28031 TestClusterScopeQuotaThrottle is still failing with broken WAL writer (#5529)(#5539)(#5582)

Limit the scope for EnvironmentEdge injection
Add timeout for ThrottleQuotaTestUtil.triggerCacheRefresh

Signed-off-by: Guanghao Zhang <[email protected]>
Signed-off-by: Bryan Beaudreault <[email protected]>
(cherry picked from commit b1eccb3)

Author: Apache9

hbase-common/src/test/java/org/apache/hadoop/hbase/util/EnvironmentEdgeManagerTestHelper.java

@@ -33,4 +33,36 @@ public static void reset() {
   public static void injectEdge(EnvironmentEdge edge) {
     EnvironmentEdgeManager.injectEdge(edge);
   }
+
+  private static final class PackageEnvironmentEdgeWrapper implements EnvironmentEdge {
+
+    private final EnvironmentEdge delegate;
+
+    private final String packageName;
+
+    PackageEnvironmentEdgeWrapper(EnvironmentEdge delegate, String packageName) {
+      this.delegate = delegate;
+      this.packageName = packageName;
+    }
+
+    @Override
+    public long currentTime() {
+      StackTraceElement[] elements = new Exception().getStackTrace();
+      // the first element is us, the second one is EnvironmentEdgeManager, so let's check the third
+      // one
+      if (elements.length > 2 && elements[2].getClassName().startsWith(packageName)) {
+        return delegate.currentTime();
+      } else {
+        return System.currentTimeMillis();
+      }
+    }
+  }
+
+  /**
+   * Inject a {@link EnvironmentEdge} which only takes effect when calling directly from the classes
+   * in the given package.
+   */
+  public static void injectEdgeForPackage(EnvironmentEdge edge, String packageName) {
+    injectEdge(new PackageEnvironmentEdgeWrapper(edge, packageName));
+  }
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java

@@ -21,6 +21,7 @@
 import java.util.Objects;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Table;
@@ -42,7 +43,9 @@ public final class ThrottleQuotaTestUtil {
   private final static int REFRESH_TIME = 30 * 60000;
   static {
     envEdge.setValue(EnvironmentEdgeManager.currentTime());
-    EnvironmentEdgeManagerTestHelper.injectEdge(envEdge);
+    // only active the envEdge for quotas package
+    EnvironmentEdgeManagerTestHelper.injectEdgeForPackage(envEdge,
+      ThrottleQuotaTestUtil.class.getPackage().getName());
   }
 
   private ThrottleQuotaTestUtil() {
@@ -135,51 +138,79 @@ private static void triggerCacheRefresh(HBaseTestingUtility testUtil, boolean by
       RegionServerRpcQuotaManager quotaManager =
         rst.getRegionServer().getRegionServerRpcQuotaManager();
       QuotaCache quotaCache = quotaManager.getQuotaCache();
-
       quotaCache.triggerCacheRefresh();
-      // sleep for cache update
       Thread.sleep(250);
-
-      for (TableName table : tables) {
-        quotaCache.getTableLimiter(table);
-      }
-
-      boolean isUpdated = false;
-      while (!isUpdated) {
-        quotaCache.triggerCacheRefresh();
-        isUpdated = true;
-        for (TableName table : tables) {
-          boolean isBypass = true;
-          if (userLimiter) {
-            isBypass = quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass();
+      testUtil.waitFor(60000, 250, new ExplainingPredicate<Exception>() {
+
+        @Override
+        public boolean evaluate() throws Exception {
+          boolean isUpdated = true;
+          for (TableName table : tables) {
+            if (userLimiter) {
+              boolean isUserBypass =
+                quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass();
+              if (isUserBypass != bypass) {
+                LOG.info(
+                  "User limiter for user={}, table={} not refreshed, bypass expected {}, actual {}",
+                  User.getCurrent(), table, bypass, isUserBypass);
+                envEdge.incValue(100);
+                isUpdated = false;
+                break;
+              }
+            }
+            if (tableLimiter) {
+              boolean isTableBypass = quotaCache.getTableLimiter(table).isBypass();
+              if (isTableBypass != bypass) {
+                LOG.info("Table limiter for table={} not refreshed, bypass expected {}, actual {}",
+                  table, bypass, isTableBypass);
+                envEdge.incValue(100);
+                isUpdated = false;
+                break;
+              }
+            }
+            if (nsLimiter) {
+              boolean isNsBypass =
+                quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass();
+              if (isNsBypass != bypass) {
+                LOG.info(
+                  "Namespace limiter for namespace={} not refreshed, bypass expected {}, actual {}",
+                  table.getNamespaceAsString(), bypass, isNsBypass);
+                envEdge.incValue(100);
+                isUpdated = false;
+                break;
+              }
+            }
           }
-          if (tableLimiter) {
-            isBypass &= quotaCache.getTableLimiter(table).isBypass();
+          if (rsLimiter) {
+            boolean rsIsBypass = quotaCache
+              .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY).isBypass();
+            if (rsIsBypass != bypass) {
+              LOG.info("RegionServer limiter not refreshed, bypass expected {}, actual {}", bypass,
+                rsIsBypass);
+              envEdge.incValue(100);
+              isUpdated = false;
+            }
           }
-          if (nsLimiter) {
-            isBypass &= quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass();
+          if (exceedThrottleQuota) {
+            if (quotaCache.isExceedThrottleQuotaEnabled() != bypass) {
+              LOG.info("ExceedThrottleQuotaEnabled not refreshed, bypass expected {}, actual {}",
+                bypass, quotaCache.isExceedThrottleQuotaEnabled());
+              envEdge.incValue(100);
+              isUpdated = false;
+            }
           }
-          if (isBypass != bypass) {
-            envEdge.incValue(100);
-            isUpdated = false;
-            break;
+          if (isUpdated) {
+            return true;
           }
+          quotaCache.triggerCacheRefresh();
+          return false;
         }
-        if (rsLimiter) {
-          boolean rsIsBypass = quotaCache
-            .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY).isBypass();
-          if (rsIsBypass != bypass) {
-            envEdge.incValue(100);
-            isUpdated = false;
-          }
-        }
-        if (exceedThrottleQuota) {
-          if (quotaCache.isExceedThrottleQuotaEnabled() != bypass) {
-            envEdge.incValue(100);
-            isUpdated = false;
-          }
+
+        @Override
+        public String explainFailure() throws Exception {
+          return "Quota cache is still not refreshed";
         }
-      }
+      });
 
       LOG.debug("QuotaCache");
       LOG.debug(Objects.toString(quotaCache.getNamespaceQuotaCache()));