From 04f4a18d84c7cf56abdef9d40c4125e21707530f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 27 Jun 2018 16:37:16 -0400 Subject: [PATCH 1/6] Test: Do not remove xpack templates when cleaning At the end of every `ESRestTestCase` we clean the cluster which includes deleting all of the templates. If xpack is installed it'll automatically recreate a few templates every time they are removed. Which is slow. This change stops the cleanup from removing the xpack templates. It cuts the time to run the docs tests more than in half and it probably saves a bit more time on other tests as well. --- .../test/rest/ESRestTestCase.java | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 495df4aa461a9..c4be8a794757a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -30,6 +30,7 @@ import org.apache.http.message.BasicHeader; import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.http.ssl.SSLContexts; +import org.apache.http.util.EntityUtils; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; @@ -67,6 +68,7 @@ import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -81,6 +83,7 @@ import static java.util.Collections.unmodifiableList; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.RETURNS_DEFAULTS; /** * Superclass for tests that interact with an external test cluster using Elasticsearch's {@link RestClient}. @@ -258,7 +261,7 @@ private void wipeCluster() throws IOException { if (preserveIndicesUponCompletion() == false) { // wipe indices try { - adminClient().performRequest("DELETE", "*"); + adminClient().performRequest(new Request("DELETE", "*")); } catch (ResponseException e) { // 404 here just means we had no indexes if (e.getResponse().getStatusLine().getStatusCode() != 404) { @@ -269,7 +272,17 @@ private void wipeCluster() throws IOException { // wipe index templates if (preserveTemplatesUponCompletion() == false) { - adminClient().performRequest("DELETE", "_template/*"); + if (hasXPack()) { + Request request = new Request("GET", "_cat/templates"); + request.addParameter("h", "name"); + String templates = EntityUtils.toString(adminClient().performRequest(request).getEntity()); + for (String template : templates.split("\n")) { + if (isXPackTemplate(template)) continue; + adminClient().performRequest(new Request("DELETE", "_template/" + template)); + } + } else { + adminClient().performRequest(new Request("DELETE", "_template/*")); + } } wipeSnapshots(); @@ -577,4 +590,31 @@ protected static Map getAsMap(final String endpoint) throws IOEx assertNotNull(responseEntity); return responseEntity; } + + /** + * Is this template one that is automatically created by xpack? Deleting + * these templates doesn't hurt anything but it makes the tests run more + * slowly because it then has to wait for the templates to be recreated. + */ + private static boolean isXPackTemplate(String name) { + if (name.startsWith(".monitoring-")) { + return true; + } + if (name.startsWith(".watch-history-")) { + return true; + } + if (name.startsWith(".ml-")) { + return true; + } + switch (name) { + case ".triggered_watches": + case ".watches": + case "logstash-index-template": + case "security_audit_log": + return true; + default: + return false; + } + } + } From b7546ef19da955ae81fcecab3a746a0c850f9161 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 28 Jun 2018 09:10:10 -0400 Subject: [PATCH 2/6] Explain --- .../java/org/elasticsearch/test/rest/ESRestTestCase.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index c4be8a794757a..2363e011311f2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -273,6 +273,12 @@ private void wipeCluster() throws IOException { // wipe index templates if (preserveTemplatesUponCompletion() == false) { if (hasXPack()) { + /* + * Delete only templates that xpack doesn't automatically + * recreate. Deleting them doesn't hurt anything, but it + * slows down the test because xpack will just recreate + * them. + */ Request request = new Request("GET", "_cat/templates"); request.addParameter("h", "name"); String templates = EntityUtils.toString(adminClient().performRequest(request).getEntity()); From 99950adf42b8cfcb92fc268f0d8ec353a611bb08 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 28 Jun 2018 09:29:09 -0400 Subject: [PATCH 3/6] Unused imports --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 2363e011311f2..3ec67d231319a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -68,7 +68,6 @@ import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -83,7 +82,6 @@ import static java.util.Collections.unmodifiableList; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.RETURNS_DEFAULTS; /** * Superclass for tests that interact with an external test cluster using Elasticsearch's {@link RestClient}. From 9e16d0bd8c502beb7e06403068696a22b6e107ef Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 2 Jul 2018 11:11:22 -0400 Subject: [PATCH 4/6] Fix up comment --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 48755061068c3..f6717502b10eb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -604,9 +604,7 @@ protected static Map getAsMap(final String endpoint) throws IOEx } /** - * Is this template one that is automatically created by xpack? Deleting - * these templates doesn't hurt anything but it makes the tests run more - * slowly because it then has to wait for the templates to be recreated. + * Is this template one that is automatically created by xpack? */ private static boolean isXPackTemplate(String name) { if (name.startsWith(".monitoring-")) { From 09cffbe0d036ae2b4bed2c231a192d9bebe6d5a4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 3 Jul 2018 10:26:55 -0400 Subject: [PATCH 5/6] Never try and clean templates named empty string --- .../org/elasticsearch/test/rest/ESRestTestCase.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index f6717502b10eb..4c3c8f1fef6dd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -281,11 +281,18 @@ private void wipeCluster() throws IOException { Request request = new Request("GET", "_cat/templates"); request.addParameter("h", "name"); String templates = EntityUtils.toString(adminClient().performRequest(request).getEntity()); - for (String template : templates.split("\n")) { - if (isXPackTemplate(template)) continue; - adminClient().performRequest(new Request("DELETE", "_template/" + template)); + if (false == "".equals(templates)) { + for (String template : templates.split("\n")) { + if (isXPackTemplate(template)) continue; + if ("".equals(templates)) { + throw new IllegalStateException("empty template in templates list:\n" + templates); + } + logger.debug("Clearing template [{}]", template); + adminClient().performRequest(new Request("DELETE", "_template/" + template)); + } } } else { + logger.debug("Clearing all templates"); adminClient().performRequest(new Request("DELETE", "_template/*")); } } From 888db579bf15b60d10b270d02d56d49ac72d91ed Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 3 Jul 2018 10:51:21 -0400 Subject: [PATCH 6/6] Fix error catching --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 4c3c8f1fef6dd..81a9598496bf5 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -284,7 +284,7 @@ private void wipeCluster() throws IOException { if (false == "".equals(templates)) { for (String template : templates.split("\n")) { if (isXPackTemplate(template)) continue; - if ("".equals(templates)) { + if ("".equals(template)) { throw new IllegalStateException("empty template in templates list:\n" + templates); } logger.debug("Clearing template [{}]", template);