From 08609418985c3cc90c6830fccac60a9c5ccc19be Mon Sep 17 00:00:00 2001
From: David Roberts <dave.roberts@elastic.co>
Date: Mon, 18 Feb 2019 12:13:21 +0000
Subject: [PATCH] [ML] More advanced post-test cleanup of ML indices

The .ml-annotations index is created asynchronously when
some other ML index exists.  This can interfere with the
post-test index deletion, as the .ml-annotations index
can be created after all other indices have been deleted.

This change adds an ML specific post-test cleanup step
that runs before the main cleanup and:

1. Checks if any ML indices exist
2. If so, waits for the .ml-annotations index to exist
3. Deletes the other ML indices found in step 1.
4. Calls the super class cleanup

This means that by the time the main post-test index
cleanup code runs:

1. The only ML index it has to delete will be the
   .ml-annotations index
2. No other ML indices will exist that could trigger
   recreation of the .ml-annotations index

Fixes #38952
---
 .../xpack/ml/MlSingleNodeTestCase.java        | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
index 6c14423d9acdb..886e0e9b57588 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java
@@ -15,11 +15,14 @@
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.ml.MachineLearningField;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
 /**
  * An extension to {@link ESSingleNodeTestCase} that adds node settings specifically needed for ML test cases.
  */
@@ -46,6 +49,33 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(LocalStateMachineLearning.class);
     }
 
+    /**
+     * This cleanup is to fix the problem described in
+     * https://github.com/elastic/elasticsearch/issues/38952
+     */
+    @Override
+    public void tearDown() throws Exception {
+        try {
+            logger.info("[{}#{}]: ML-specific after test cleanup", getTestClass().getSimpleName(), getTestName());
+            String[] nonAnnotationMlIndices;
+            boolean mlAnnotationsIndexExists;
+            do {
+                String[] mlIndices = client().admin().indices().prepareGetIndex().addIndices(".ml-*").get().indices();
+                nonAnnotationMlIndices = Arrays.stream(mlIndices).filter(name -> name.startsWith(".ml-annotations") == false)
+                    .toArray(String[]::new);
+                mlAnnotationsIndexExists = mlIndices.length > nonAnnotationMlIndices.length;
+            } while (nonAnnotationMlIndices.length > 0 && mlAnnotationsIndexExists == false);
+            if (nonAnnotationMlIndices.length > 0) {
+                // Delete the ML indices apart from the annotations index.  The annotations index will be deleted by the
+                // base class cleanup.  We want to delete all the others first so that the annotations index doesn't get
+                // automatically recreated.
+                assertAcked(client().admin().indices().prepareDelete(nonAnnotationMlIndices).get());
+            }
+        } finally {
+            super.tearDown();
+        }
+    }
+
     protected void waitForMlTemplates() throws Exception {
         // block until the templates are installed
         assertBusy(() -> {