[ILM] Fix issue where index may not yet be in 'hot' phase (#35716)

It's possible that we check the explain status so quickly after the index is
managed that it is still in the "new" phase, having not yet moved to the "hot"
phase.

This commit fixes that by wrapping the checks in `assertBusy`

Resolves #35690

Author: dakrone

client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java

@@ -183,30 +183,36 @@ public void testExplainLifecycle() throws Exception {
         createIndex("squash", Settings.EMPTY);
 
         ExplainLifecycleRequest req = new ExplainLifecycleRequest("foo-01", "baz-01", "squash");
-        ExplainLifecycleResponse response = execute(req, highLevelClient().indexLifecycle()::explainLifecycle,
+        // Occasionally the explain is so fast that the indices have not yet left the "new" phase,
+        // so we assertBusy here in the event that they haven't progressed to the "hot" phase yet.
+        assertBusy(() -> {
+            ExplainLifecycleResponse response = execute(req, highLevelClient().indexLifecycle()::explainLifecycle,
                 highLevelClient().indexLifecycle()::explainLifecycleAsync);
-        Map<String, IndexLifecycleExplainResponse> indexResponses = response.getIndexResponses();
-        assertEquals(3, indexResponses.size());
-        IndexLifecycleExplainResponse fooResponse = indexResponses.get("foo-01");
-        assertNotNull(fooResponse);
-        assertTrue(fooResponse.managedByILM());
-        assertEquals("foo-01", fooResponse.getIndex());
-        assertEquals("hot", fooResponse.getPhase());
-        assertEquals("rollover", fooResponse.getAction());
-        assertEquals("check-rollover-ready", fooResponse.getStep());
-        assertEquals(new PhaseExecutionInfo(policy.getName(), new Phase("", hotPhase.getMinimumAge(), hotPhase.getActions()),
+            Map<String, IndexLifecycleExplainResponse> indexResponses = response.getIndexResponses();
+            assertEquals(3, indexResponses.size());
+
+            IndexLifecycleExplainResponse squashResponse = indexResponses.get("squash");
+            assertNotNull(squashResponse);
+            assertFalse(squashResponse.managedByILM());
+            assertEquals("squash", squashResponse.getIndex());
+
+            IndexLifecycleExplainResponse fooResponse = indexResponses.get("foo-01");
+            assertNotNull(fooResponse);
+            assertTrue(fooResponse.managedByILM());
+            assertEquals("foo-01", fooResponse.getIndex());
+            assertEquals("hot", fooResponse.getPhase());
+            assertEquals("rollover", fooResponse.getAction());
+            assertEquals("check-rollover-ready", fooResponse.getStep());
+            assertEquals(new PhaseExecutionInfo(policy.getName(), new Phase("", hotPhase.getMinimumAge(), hotPhase.getActions()),
                 1L, expectedPolicyModifiedDate), fooResponse.getPhaseExecutionInfo());
-        IndexLifecycleExplainResponse bazResponse = indexResponses.get("baz-01");
-        assertNotNull(bazResponse);
-        assertTrue(bazResponse.managedByILM());
-        assertEquals("baz-01", bazResponse.getIndex());
-        assertEquals("hot", bazResponse.getPhase());
-        assertEquals("rollover", bazResponse.getAction());
-        assertEquals("check-rollover-ready", bazResponse.getStep());
-        IndexLifecycleExplainResponse squashResponse = indexResponses.get("squash");
-        assertNotNull(squashResponse);
-        assertFalse(squashResponse.managedByILM());
-        assertEquals("squash", squashResponse.getIndex());
+            IndexLifecycleExplainResponse bazResponse = indexResponses.get("baz-01");
+            assertNotNull(bazResponse);
+            assertTrue(bazResponse.managedByILM());
+            assertEquals("baz-01", bazResponse.getIndex());
+            assertEquals("hot", bazResponse.getPhase());
+            assertEquals("rollover", bazResponse.getAction());
+            assertEquals("check-rollover-ready", bazResponse.getStep());
+        });
     }
 
     public void testDeleteLifecycle() throws IOException {