From 253b7e079405345a6aa87c6a89bd286b842d611a Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 26 Sep 2019 11:26:00 +0100 Subject: [PATCH] [TEST] Work around _cat/indices bug with security enabled When the ML native multi-node tests use _cat/indices/_all and the request goes to a non-master node, _all is translated to a list of concrete indices on the coordinating node before the request is forwarded to the master node. Then it is possible for the master node to return an index_not_found_exception if one of the concrete indices that was expanded on the coordinating node has been deleted in the meantime. It has been observed that the index that gets deleted when the problem affects the ML native multi-node tests is always the ML notifications index. The tests that fail are only interested in the presence or absense of ML results indices. Therefore the workaround is to only _cat indices that match the ML results patten. Fixes #45652 --- .../xpack/ml/integration/MlJobIT.java | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 1c7a367239ed2..cfd760df92679 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -208,7 +208,9 @@ public void testCreateJobsWithIndexNameOption() throws Exception { } }); - String responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 + String responseAsString = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); @@ -272,7 +274,8 @@ public void testCreateJobsWithIndexNameOption() throws Exception { assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId1)))); assertThat(responseAsString, containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2))); //job2 still exists - responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + responseAsString = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(responseAsString, containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName)); client().performRequest(new Request("POST", "/_refresh")); @@ -287,7 +290,8 @@ public void testCreateJobsWithIndexNameOption() throws Exception { assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsAliasedName(jobId2)))); client().performRequest(new Request("POST", "/_refresh")); - responseAsString = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + responseAsString = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(responseAsString, not(containsString(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-" + indexName))); } @@ -394,19 +398,21 @@ public void testCreateJob_WithClashingFieldMappingsFails() throws Exception { "avoid the clash by assigning a dedicated results index")); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/45652") public void testDeleteJob() throws Exception { String jobId = "delete-job-job"; String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; createFarequoteJob(jobId); - String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 + String indicesBeforeDelete = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(indicesBeforeDelete, containsString(indexName)); client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId)); // check that the index still exists (it's shared by default) - String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + String indicesAfterDelete = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(indicesAfterDelete, containsString(indexName)); waitUntilIndexIsEmpty(indexName); @@ -465,13 +471,14 @@ public void testDeleteJob_TimingStatsDocumentIsDeleted() throws Exception { assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(404)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/45652") public void testDeleteJobAsync() throws Exception { String jobId = "delete-job-async-job"; String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; createFarequoteJob(jobId); - String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 + String indicesBeforeDelete = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(indicesBeforeDelete, containsString(indexName)); Response response = client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId @@ -483,7 +490,8 @@ public void testDeleteJobAsync() throws Exception { assertThat(EntityUtils.toString(taskResponse.getEntity()), containsString("\"acknowledged\":true")); // check that the index still exists (it's shared by default) - String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + String indicesAfterDelete = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(indicesAfterDelete, containsString(indexName)); waitUntilIndexIsEmpty(indexName); @@ -518,7 +526,9 @@ public void testDeleteJobAfterMissingIndex() throws Exception { String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; createFarequoteJob(jobId); - String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 + String indicesBeforeDelete = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(indicesBeforeDelete, containsString(indexName)); // Manually delete the index so that we can test that deletion proceeds @@ -528,7 +538,8 @@ public void testDeleteJobAfterMissingIndex() throws Exception { client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId)); // check index was deleted - String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + String indicesAfterDelete = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(indicesAfterDelete, not(containsString(aliasName))); assertThat(indicesAfterDelete, not(containsString(indexName))); @@ -598,7 +609,9 @@ public void testMultiIndexDelete() throws Exception { "}"); client().performRequest(extraIndex2); - String indicesBeforeDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + // Use _cat/indices/.ml-anomalies-* instead of _cat/indices/_all to workaround https://github.com/elastic/elasticsearch/issues/45652 + String indicesBeforeDelete = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(indicesBeforeDelete, containsString(indexName)); assertThat(indicesBeforeDelete, containsString(indexName + "-001")); assertThat(indicesBeforeDelete, containsString(indexName + "-002")); @@ -637,7 +650,8 @@ public void testMultiIndexDelete() throws Exception { client().performRequest(new Request("POST", "/_refresh")); // check that the indices still exist but are empty - String indicesAfterDelete = EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/indices")).getEntity()); + String indicesAfterDelete = EntityUtils.toString(client().performRequest( + new Request("GET", "/_cat/indices/" + AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "*")).getEntity()); assertThat(indicesAfterDelete, containsString(indexName)); assertThat(indicesAfterDelete, containsString(indexName + "-001")); assertThat(indicesAfterDelete, containsString(indexName + "-002"));