Retry on RepositoryException in SLM tests (#48548)

Due to a bug, GETing a snapshot can cause a RespositoryException to be
thrown. This error is transient and should be retried, rather than
causing the test to fail. This commit converts those
RepositoryExceptions into AssertionErrors so that they will be retried
in code wrapped in assertBusy.

Author: gwbrown

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java

@@ -1056,6 +1056,8 @@ private void assertSnapshotExists(final RestHighLevelClient client, final String
             } catch (Exception e) {
                 if (e.getMessage().contains("snapshot_missing_exception")) {
                     fail("snapshot does not exist: " + snapshotName);
+                } else if (e.getMessage().contains("repository_exception")) {
+                    fail("got a respository_exception, retrying. original message: " + e.getMessage());
                 }
                 throw e;
             }

x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.snapshots.ConcurrentSnapshotExecutionException;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotMissingException;
@@ -176,8 +177,7 @@ public void testRetentionWhileSnapshotInProgress() throws Exception {
         logger.info("--> kicked off snapshot {}", completedSnapshotName);
         assertBusy(() -> {
             try {
-                SnapshotsStatusResponse s =
-                    client().admin().cluster().prepareSnapshotStatus(REPO).setSnapshots(completedSnapshotName).get();
+                SnapshotsStatusResponse s = getSnapshotStatus(completedSnapshotName);
                 assertThat("expected a snapshot but none were returned", s.getSnapshots().size(), equalTo(1));
                 SnapshotStatus status = s.getSnapshots().get(0);
                 logger.info("--> waiting for snapshot {} to be completed, got: {}", completedSnapshotName, status.getState());
@@ -245,8 +245,7 @@ public void testRetentionWhileSnapshotInProgress() throws Exception {
                 client().admin().cluster().prepareReroute().get();
                 logger.info("--> waiting for snapshot to be deleted");
                 try {
-                    SnapshotsStatusResponse s =
-                        client().admin().cluster().prepareSnapshotStatus(REPO).setSnapshots(completedSnapshotName).get();
+                    SnapshotsStatusResponse s = getSnapshotStatus(completedSnapshotName);
                     assertNull("expected no snapshot but one was returned", s.getSnapshots().get(0));
                 } catch (SnapshotMissingException e) {
                     // Great, we wanted it to be deleted!
@@ -406,6 +405,18 @@ private void testUnsuccessfulSnapshotRetention(boolean partialSuccess) throws Ex
         }
     }
 
+    private SnapshotsStatusResponse getSnapshotStatus(String snapshotName) {
+        try {
+            return client().admin().cluster().prepareSnapshotStatus(REPO).setSnapshots(snapshotName).get();
+        } catch (RepositoryException e) {
+            // Convert this to an AssertionError so that it can be retried in an assertBusy - this is often a transient error because
+            // concurrent status calls and write operations may lead to failures in determining the current repository generation
+            // TODO: Remove this hack once tracking the current repository generation has been made consistent
+            logger.warn(e);
+            throw new AssertionError(e);
+        }
+    }
+
     private void createAndPopulateIndex(String indexName) throws InterruptedException {
         logger.info("--> creating and populating index [{}]", indexName);
         assertAcked(prepareCreate(indexName, 0, Settings.builder()