Wait for cluster to become quiescent between REST tests (#24148)

[TEST] ensures REST tests wait for cluster state updates to finish
processing before moving to the next test

Author: Ali Beyad

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -125,6 +125,7 @@ public void initClient() throws IOException {
     @After
     public final void cleanUpCluster() throws Exception {
         wipeCluster();
+        waitForClusterStateUpdatesToFinish();
         logIfThereAreRunningTasks();
     }
 
@@ -253,6 +254,22 @@ private void logIfThereAreRunningTasks() throws InterruptedException, IOExceptio
          */
     }
 
+    /**
+     * Waits for the cluster state updates to have been processed, so that no cluster
+     * state updates are still in-progress when the next test starts.
+     */
+    private void waitForClusterStateUpdatesToFinish() throws Exception {
+        assertBusy(() -> {
+            try {
+                Response response = adminClient().performRequest("GET", "_cluster/pending_tasks");
+                List<Object> tasks = (List<Object>) entityAsMap(response).get("tasks");
+                assertTrue(tasks.isEmpty());
+            } catch (IOException e) {
+                fail("cannot get cluster's pending tasks: " + e.getMessage());
+            }
+        });
+    }
+
     /**
      * Used to obtain settings for the REST client that is used to send REST requests.
      */