HBASE-25032 Wait for region server to become online before adding it to online servers in Master

Author: caroliney14

hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java

@@ -204,7 +204,8 @@ public boolean unregisterListener(final ServerListener listener) {
   }
 
   /**
-   * Let the server manager know a new regionserver has come online
+   * Let the server manager know a regionserver is requesting configurations.
+   * Regionserver will not be added here, but in its first report.
    * @param request the startup request
    * @param versionNumber the version number of the new regionserver
    * @param version the version of the new regionserver, could contain strings like "SNAPSHOT"
@@ -227,10 +228,6 @@ ServerName regionServerStartup(RegionServerStartupRequest request, int versionNu
     ServerName sn = ServerName.valueOf(hostname, request.getPort(), request.getServerStartCode());
     checkClockSkew(sn, request.getServerCurrentTime());
     checkIsDead(sn, "STARTUP");
-    if (!checkAndRecordNewServer(sn, ServerMetricsBuilder.of(sn, versionNumber, version))) {
-      LOG.warn(
-        "THIS SHOULD NOT HAPPEN, RegionServerStartup" + " could not record the server: " + sn);
-    }
     return sn;
   }
 
@@ -281,7 +278,6 @@ public void regionServerReport(ServerName sn,
     if (null == this.onlineServers.replace(sn, sl)) {
       // Already have this host+port combo and its just different start code?
       // Just let the server in. Presume master joining a running cluster.
-      // recordNewServer is what happens at the end of reportServerStartup.
       // The only thing we are skipping is passing back to the regionserver
       // the ServerName to use. Here we presume a master has already done
       // that so we'll press on with whatever it gave us for ServerName.

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -1014,8 +1014,8 @@ public void run() {
         // node was created, in case any coprocessors want to use ZooKeeper
         this.rsHost = new RegionServerCoprocessorHost(this, this.conf);
 
-        // Try and register with the Master; tell it we are here.  Break if server is stopped or
-        // the clusterup flag is down or hdfs went wacky. Once registered successfully, go ahead and
+        // Get configurations from the Master. Break if server is stopped or
+        // the clusterup flag is down or hdfs went wacky. Then start up all Services.
         // start up all Services. Use RetryCounter to get backoff in case Master is struggling to
         // come up.
         LOG.debug("About to register with Master.");
@@ -1050,7 +1050,7 @@ public void run() {
         }
       }
 
-      // We registered with the Master.  Go into run mode.
+      // Run mode.
       long lastMsg = System.currentTimeMillis();
       long oldRequestCount = -1;
       // The main run loop.
@@ -1084,7 +1084,14 @@ public void run() {
         }
         long now = System.currentTimeMillis();
         if ((now - lastMsg) >= msgInterval) {
-          tryRegionServerReport(lastMsg, now);
+          // Register with the Master now that our setup is complete.
+          if (tryRegionServerReport(lastMsg, now) && !online.get()) {
+            // Wake up anyone waiting for this server to online
+            synchronized (online) {
+              online.set(true);
+              online.notifyAll();
+            }
+          }
           lastMsg = System.currentTimeMillis();
         }
         if (!isStopped() && !isAborted()) {
@@ -1253,12 +1260,12 @@ private long getWriteRequestCount() {
   }
 
   @InterfaceAudience.Private
-  protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
+  protected boolean tryRegionServerReport(long reportStartTime, long reportEndTime)
       throws IOException {
     RegionServerStatusService.BlockingInterface rss = rssStub;
     if (rss == null) {
       // the current server could be stopping.
-      return;
+      return false;
     }
     ClusterStatusProtos.ServerLoad sl = buildServerLoad(reportStartTime, reportEndTime);
     try {
@@ -1278,7 +1285,9 @@ protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
       // Couldn't connect to the master, get location from zk and reconnect
       // Method blocks until new master is found or we are stopped
       createRegionServerStatusStub(true);
+      return false;
     }
+    return true;
   }
 
   /**
@@ -1653,11 +1662,6 @@ protected void handleReportForDutyResponse(final RegionServerStartupResponse c)
           ", sessionid=0x" +
           Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()));
 
-      // Wake up anyone waiting for this server to online
-      synchronized (online) {
-        online.set(true);
-        online.notifyAll();
-      }
     } catch (Throwable e) {
       stop("Failed initialization");
       throw convertThrowableToIOE(cleanup(e, "Failed init"),
@@ -2836,10 +2840,10 @@ private boolean keepLooping() {
   }
 
   /*
-   * Let the master know we're here Run initialization using parameters passed
+   * Run initialization using parameters passed us by the master.
    * us by the master.
    * @return A Map of key/value configurations we got from the Master else
-   * null if we failed to register.
+   * null if we failed during report.
    * @throws IOException
    */
   private RegionServerStartupResponse reportForDuty() throws IOException {

hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java

@@ -446,8 +446,9 @@ public JVMClusterUtil.RegionServerThread startRegionServerAndWait(long timeout)
     ServerName rsServerName = t.getRegionServer().getServerName();
 
     long start = System.currentTimeMillis();
-    ClusterMetrics clusterStatus = getClusterMetrics();
+    ClusterMetrics clusterStatus;
     while ((System.currentTimeMillis() - start) < timeout) {
+      clusterStatus = getClusterMetrics();
       if (clusterStatus != null && clusterStatus.getLiveServerMetrics().containsKey(rsServerName)) {
         return t;
       }

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestGetReplicationLoad.java

@@ -58,8 +58,9 @@ public MyMaster(Configuration conf) throws IOException, KeeperException, Interru
     }
 
     @Override
-    protected void tryRegionServerReport(long reportStartTime, long reportEndTime) {
+    protected boolean tryRegionServerReport(long reportStartTime, long reportEndTime) {
       // do nothing
+      return true;
     }
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java

@@ -61,23 +61,13 @@ public static class MyMaster extends HMaster {
     public MyMaster(Configuration conf) throws IOException, KeeperException, InterruptedException {
       super(conf);
     }
-
-    @Override
-    protected void tryRegionServerReport(long reportStartTime, long reportEndTime) {
-      // do nothing
-    }
   }
 
   public static class MyRegionServer extends MiniHBaseCluster.MiniHBaseClusterRegionServer {
 
     public MyRegionServer(Configuration conf) throws IOException, InterruptedException {
       super(conf);
     }
-
-    @Override
-    protected void tryRegionServerReport(long reportStartTime, long reportEndTime) {
-      // do nothing
-    }
   }
 
   @BeforeClass
@@ -108,11 +98,14 @@ public void testClusterRequests() throws Exception {
     request.setServer(ProtobufUtil.toServerName(serverName));
     long expectedRequestNumber = 10000;
 
-    MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
     ClusterStatusProtos.ServerLoad sl = ClusterStatusProtos.ServerLoad.newBuilder()
       .setTotalNumberOfRequests(expectedRequestNumber).build();
     request.setLoad(sl);
 
+    MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
+    // Init master source again to reset cluster requests counter
+    masterSource.init();
+
     master.getMasterRpcServices().regionServerReport(null, request.build());
     metricsHelper.assertCounter("cluster_requests", expectedRequestNumber, masterSource);
 

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionInDeadRegionServer.java

@@ -84,13 +84,14 @@ public IgnoreYouAreDeadRS(Configuration conf) throws IOException, InterruptedExc
     }
 
     @Override
-    protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
+    protected boolean tryRegionServerReport(long reportStartTime, long reportEndTime)
         throws IOException {
       try {
         super.tryRegionServerReport(reportStartTime, reportEndTime);
       } catch (YouAreDeadException e) {
         // ignore, do not abort
       }
+      return true;
     }
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestShutdownWhileWALBroken.java

@@ -85,13 +85,15 @@ public MyRegionServer(Configuration conf) throws IOException {
     }
 
     @Override
-    protected void tryRegionServerReport(long reportStartTime, long reportEndTime)
+    protected boolean tryRegionServerReport(long reportStartTime, long reportEndTime)
         throws IOException {
       try {
         super.tryRegionServerReport(reportStartTime, reportEndTime);
       } catch (YouAreDeadException e) {
         LOG.info("Caught YouAreDeadException, ignore", e);
+        return false;
       }
+      return true;
     }
 
     @Override