Unmute and increase logging on flaky SLM tests (#48612)

The failures in these tests have been remarkably difficult to track
down, in part because they will not reproduce locally. This commit
unmutes the flaky tests and increases logging, as well as introducing
some additional logging, to attempt to pin down the failures.

Author: gwbrown

x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/slm/SnapshotLifecycleRestIT.java

@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.slm;
 
 import org.apache.http.util.EntityUtils;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -27,6 +26,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
 import org.elasticsearch.xpack.core.ilm.Step;
@@ -195,6 +195,8 @@ public void testPolicyFailure() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
+    @TestIssueLogging(value = "org.elasticsearch.xpack.slm:TRACE,org.elasticsearch.xpack.core.slm:TRACE,org.elasticsearch.snapshots:DEBUG",
+        issueUrl = "https://github.com/elastic/elasticsearch/issues/48531")
     public void testPolicyManualExecution() throws Exception {
         final String indexName = "test";
         final String policyName = "test-policy";
@@ -204,6 +206,8 @@ public void testPolicyManualExecution() throws Exception {
             index(client(), indexName, "" + i, "foo", "bar");
         }
 
+        logSLMPolicies();
+
         // Create a snapshot repo
         initializeRepo(repoId);
 
@@ -216,6 +220,8 @@ public void testPolicyManualExecution() throws Exception {
 
         final String snapshotName = executePolicy(policyName);
 
+        logSLMPolicies();
+
         // Check that the executed snapshot is created
         assertBusy(() -> {
             try {
@@ -336,7 +342,8 @@ public void testStartStopStatus() throws Exception {
     }
 
     @SuppressWarnings("unchecked")
-    @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48017")
+    @TestIssueLogging(value = "org.elasticsearch.xpack.slm:TRACE,org.elasticsearch.xpack.core.slm:TRACE,org.elasticsearch.snapshots:TRACE",
+        issueUrl = "https://github.com/elastic/elasticsearch/issues/48017")
     public void testBasicTimeBasedRetenion() throws Exception {
         final String indexName = "test";
         final String policyName = "test-policy";
@@ -346,6 +353,7 @@ public void testBasicTimeBasedRetenion() throws Exception {
         for (int i = 0; i < docCount; i++) {
             index(client(), indexName, "" + i, "foo", "bar");
         }
+        logSLMPolicies();
 
         // Create a snapshot repo
         initializeRepo(repoId);
@@ -383,8 +391,11 @@ public void testBasicTimeBasedRetenion() throws Exception {
             Request r = new Request("PUT", "/_cluster/settings");
             r.setJsonEntity(Strings.toString(builder));
             Response updateSettingsResp = client().performRequest(r);
+            assertAcked(updateSettingsResp);
         }
 
+        logSLMPolicies();
+
         try {
             // Check that the snapshot created by the policy has been removed by retention
             assertBusy(() -> {
@@ -618,7 +629,8 @@ private void createSnapshotPolicy(String policyName, String snapshotNamePattern,
         XContentBuilder lifecycleBuilder = JsonXContent.contentBuilder();
         policy.toXContent(lifecycleBuilder, ToXContent.EMPTY_PARAMS);
         putLifecycle.setJsonEntity(Strings.toString(lifecycleBuilder));
-        assertOK(client().performRequest(putLifecycle));
+        final Response response = client().performRequest(putLifecycle);
+        assertAcked(response);
     }
 
     private void initializeRepo(String repoName) throws IOException {
@@ -659,4 +671,20 @@ private static Map<String, Object> policyStatsAsMap(Map<String, Object> stats) {
                 m -> (String) m.get(SnapshotLifecycleStats.SnapshotPolicyStats.POLICY_ID.getPreferredName()),
                 Function.identity()));
     }
+
+    private void assertAcked(Response response) throws IOException {
+        assertOK(response);
+        Map<String, Object> putLifecycleResponseMap;
+        try (InputStream is = response.getEntity().getContent()) {
+            putLifecycleResponseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true);
+        }
+        assertThat(putLifecycleResponseMap.get("acknowledged"), equalTo(true));
+    }
+
+    private void logSLMPolicies() throws IOException {
+        Request request = new Request("GET" , "/_slm/policy?human");
+        Response response = client().performRequest(request);
+        assertOK(response);
+        logger.info("SLM policies: {}", EntityUtils.toString(response.getEntity()));
+    }
 }