From 0f425e01a0566bb0594b814f66b8b64f28218148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Wed, 8 Sep 2021 12:53:46 +0200 Subject: [PATCH 01/18] Make ML indices hidden when the node becomes master --- .../MlInitializationServiceIT.java | 103 ++++++++++++++++++ .../xpack/ml/MachineLearning.java | 16 ++- .../xpack/ml/MlDailyMaintenanceService.java | 6 +- .../xpack/ml/MlInitializationService.java | 46 +++++++- .../ml/MlInitializationServiceTests.java | 22 +++- 5 files changed, 184 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java new file mode 100644 index 0000000000000..87ae4f942e589 --- /dev/null +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ml.MlDailyMaintenanceService; +import org.elasticsearch.xpack.ml.MlInitializationService; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collections; +import java.util.stream.Stream; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MlInitializationServiceIT extends MlNativeAutodetectIntegTestCase { + + private ThreadPool threadPool; + private MlInitializationService mlInitializationService; + + @Before + public void setUpMocks() { + threadPool = mock(ThreadPool.class); + when(threadPool.executor(ThreadPool.Names.SAME)).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + MlDailyMaintenanceService mlDailyMaintenanceService = mock(MlDailyMaintenanceService.class); + mlInitializationService = new MlInitializationService(client(), threadPool, mlDailyMaintenanceService, clusterService()); + } + + public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() { + String[] mlHiddenIndexNames = { + ".ml-anomalies-7", + ".ml-state-000001", + ".ml-stats-000001", + ".ml-notifications-000002", + ".ml-annotations-6" + }; + String[] otherIndexNames = { "some-index-1", "some-other-index-2" }; + String[] allIndexNames = Stream.concat(Arrays.stream(mlHiddenIndexNames), Arrays.stream(otherIndexNames)).toArray(String[]::new); + + for (String indexName : mlHiddenIndexNames) { + try { + assertAcked(prepareCreate(indexName).setSettings(Collections.singletonMap(SETTING_INDEX_HIDDEN, randomBoolean()))); + } catch (ResourceAlreadyExistsException e) { + logger.info("Index " + indexName + "already exists: {}", e.getDetailedMessage()); + } + } + createIndex(otherIndexNames); + + GetSettingsResponse settingsResponse = + client().admin().indices().prepareGetSettings(allIndexNames) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) + .get(); + assertThat(settingsResponse, is(notNullValue())); + for (String indexName : mlHiddenIndexNames) { + Settings settings = settingsResponse.getIndexToSettings().get(indexName); + assertThat(settings, is(notNullValue())); + } + for (String indexName : otherIndexNames) { + Settings settings = settingsResponse.getIndexToSettings().get(indexName); + assertThat(settings, is(notNullValue())); + assertThat( + "Index " + indexName + " expected not to be hidden but was", + settings.getAsBoolean(SETTING_INDEX_HIDDEN, false), is(equalTo(false))); + } + + mlInitializationService.onMaster(); + + settingsResponse = + client().admin().indices().prepareGetSettings(allIndexNames) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) + .get(); + assertThat(settingsResponse, is(notNullValue())); + for (String indexName : mlHiddenIndexNames) { + Settings settings = settingsResponse.getIndexToSettings().get(indexName); + assertThat(settings, is(notNullValue())); + assertThat( + "Index " + indexName + " expected to be hidden but wasn't", + settings.getAsBoolean(SETTING_INDEX_HIDDEN, false), is(equalTo(true))); + } + for (String indexName : otherIndexNames) { + Settings settings = settingsResponse.getIndexToSettings().get(indexName); + assertThat(settings, is(notNullValue())); + assertThat( + "Index " + indexName + " expected not to be hidden but was", + settings.getAsBoolean(SETTING_INDEX_HIDDEN, false), is(equalTo(false))); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index bf3b71b0eb93d..8bcce63145988 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1419,15 +1419,25 @@ public static SystemIndexDescriptor getInferenceIndexSecurityDescriptor() { .build(); } - @Override - public Collection getAssociatedIndexDescriptors() { - return List.of( + private static Collection ASSOCIATED_INDEX_DESCRIPTORS = + List.of( new AssociatedIndexDescriptor(RESULTS_INDEX_PREFIX + "*", "Results indices"), new AssociatedIndexDescriptor(STATE_INDEX_PREFIX + "*", "State indices"), new AssociatedIndexDescriptor(MlStatsIndex.indexPattern(), "ML stats index"), new AssociatedIndexDescriptor(".ml-notifications*", "ML notifications indices"), new AssociatedIndexDescriptor(".ml-annotations*", "Ml annotations indices") ); + + @Override + public Collection getAssociatedIndexDescriptors() { + return ASSOCIATED_INDEX_DESCRIPTORS; + } + + public static String[] getMlHiddenIndexPatterns() { + return ASSOCIATED_INDEX_DESCRIPTORS + .stream() + .map(AssociatedIndexDescriptor::getIndexPattern) + .toArray(String[]::new); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index 5c94a8a5a0d78..e3f5c6b331187 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -18,12 +18,12 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.core.Releasable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index b51df785b6e09..4da7f97dd8776 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -9,6 +9,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -20,14 +23,21 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; -class MlInitializationService implements ClusterStateListener { +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; + +public class MlInitializationService implements ClusterStateListener { private static final Logger logger = LogManager.getLogger(MlInitializationService.class); private final Client client; + private final ThreadPool threadPool; private final AtomicBoolean isIndexCreationInProgress = new AtomicBoolean(false); private final MlDailyMaintenanceService mlDailyMaintenanceService; @@ -37,6 +47,7 @@ class MlInitializationService implements ClusterStateListener { MlInitializationService(Settings settings, ThreadPool threadPool, ClusterService clusterService, Client client, MlAssignmentNotifier mlAssignmentNotifier) { this(client, + threadPool, new MlDailyMaintenanceService( settings, Objects.requireNonNull(clusterService).getClusterName(), @@ -49,8 +60,10 @@ class MlInitializationService implements ClusterStateListener { } // For testing - MlInitializationService(Client client, MlDailyMaintenanceService dailyMaintenanceService, ClusterService clusterService) { + public MlInitializationService(Client client, ThreadPool threadPool, MlDailyMaintenanceService dailyMaintenanceService, + ClusterService clusterService) { this.client = Objects.requireNonNull(client); + this.threadPool = threadPool; this.mlDailyMaintenanceService = dailyMaintenanceService; clusterService.addListener(this); clusterService.addLifecycleListener(new LifecycleListener() { @@ -71,6 +84,7 @@ public void beforeStop() { public void onMaster() { mlDailyMaintenanceService.start(); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(this::makeMlIndicesHidden); } public void offMaster() { @@ -112,5 +126,33 @@ MlDailyMaintenanceService getDailyMaintenanceService() { return mlDailyMaintenanceService; } + private void makeMlIndicesHidden() { + String[] mlHiddenIndexPatterns = MachineLearning.getMlHiddenIndexPatterns(); + GetSettingsResponse getSettingsResponse = + client.admin().indices().prepareGetSettings() + .setIndices(mlHiddenIndexPatterns) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) + .get(); + String[] nonHiddenIndices = + getSettingsResponse.getIndexToSettings().stream() + .filter(e -> e.getValue().getAsBoolean(SETTING_INDEX_HIDDEN, false) == false) + .map(Map.Entry::getKey) + .toArray(String[]::new); + if (nonHiddenIndices.length == 0) { + logger.info("There are no indices that need to be made hidden, " + getSettingsResponse); + return; + } + String nonHiddenIndicesString = Arrays.stream(nonHiddenIndices).collect(Collectors.joining(", ")); + logger.info("The following indices will now be made hidden: {}", nonHiddenIndicesString); + AcknowledgedResponse updateSettingsResponse = + client.admin().indices().prepareUpdateSettings() + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) + .setIndices(nonHiddenIndices) + .setSettings(Collections.singletonMap(SETTING_INDEX_HIDDEN, true)) + .get(); + if (updateSettingsResponse.isAcknowledged() == false) { + logger.error("One or more of the following indices could not be made hidden: {}", nonHiddenIndicesString); + } + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java index eed3eef2050de..9f4700117b2d1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java @@ -6,9 +6,16 @@ */ package org.elasticsearch.xpack.ml; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.Scheduler; @@ -20,6 +27,7 @@ import static org.elasticsearch.mock.orig.Mockito.doAnswer; import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -47,11 +55,22 @@ public void setUpMocks() { return null; }).when(executorService).execute(any(Runnable.class)); when(threadPool.executor(ThreadPool.Names.GENERIC)).thenReturn(executorService); + when(threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)).thenReturn(executorService); Scheduler.ScheduledCancellable scheduledCancellable = mock(Scheduler.ScheduledCancellable.class); when(threadPool.schedule(any(), any(), any())).thenReturn(scheduledCancellable); when(clusterService.getClusterName()).thenReturn(CLUSTER_NAME); + + IndicesAdminClient indicesAdminClient = mock(IndicesAdminClient.class); + when(indicesAdminClient.prepareGetSettings()).thenReturn(new GetSettingsRequestBuilder(client, GetSettingsAction.INSTANCE)); + AdminClient adminClient = mock(AdminClient.class); + when(adminClient.indices()).thenReturn(indicesAdminClient); + when(client.admin()).thenReturn(adminClient); + @SuppressWarnings("unchecked") + ActionFuture actionFuture = mock(ActionFuture.class); + when(actionFuture.actionGet()).thenReturn(new GetSettingsResponse(ImmutableOpenMap.of(), ImmutableOpenMap.of())); + when(client.execute(eq(GetSettingsAction.INSTANCE), any())).thenReturn(actionFuture); } public void testInitialize() { @@ -71,7 +90,8 @@ public void testInitialize_noMasterNode() { public void testNodeGoesFromMasterToNonMasterAndBack() { MlDailyMaintenanceService initialDailyMaintenanceService = mock(MlDailyMaintenanceService.class); - MlInitializationService initializationService = new MlInitializationService(client, initialDailyMaintenanceService, clusterService); + MlInitializationService initializationService = + new MlInitializationService(client, threadPool, initialDailyMaintenanceService, clusterService); initializationService.offMaster(); verify(initialDailyMaintenanceService).stop(); From 0843fc19505fbdaf81b8c880988255f94bc24280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Wed, 15 Sep 2021 13:18:25 +0200 Subject: [PATCH 02/18] Revert the unnecessary change --- .../elasticsearch/xpack/ml/MlDailyMaintenanceService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java index e3f5c6b331187..5c94a8a5a0d78 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlDailyMaintenanceService.java @@ -18,12 +18,12 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.core.Releasable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.Releasable; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; From 01e276a962902d7fd8bd915b0b7a8f59b6c1de00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Wed, 15 Sep 2021 13:42:54 +0200 Subject: [PATCH 03/18] Fix integration test --- .../xpack/ml/integration/MlInitializationServiceIT.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java index 87ae4f942e589..32e886f404602 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java @@ -9,9 +9,11 @@ import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.MlDailyMaintenanceService; import org.elasticsearch.xpack.ml.MlInitializationService; import org.junit.Before; @@ -37,8 +39,10 @@ public class MlInitializationServiceIT extends MlNativeAutodetectIntegTestCase { public void setUpMocks() { threadPool = mock(ThreadPool.class); when(threadPool.executor(ThreadPool.Names.SAME)).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME)).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); MlDailyMaintenanceService mlDailyMaintenanceService = mock(MlDailyMaintenanceService.class); - mlInitializationService = new MlInitializationService(client(), threadPool, mlDailyMaintenanceService, clusterService()); + ClusterService clusterService = mock(ClusterService.class); + mlInitializationService = new MlInitializationService(client(), threadPool, mlDailyMaintenanceService, clusterService); } public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() { From b42085b8b1eedfb4e6568567bdb5f29685790726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Thu, 16 Sep 2021 15:42:12 +0200 Subject: [PATCH 04/18] Add full cluster restart test --- .../MlHiddenIndicesFullClusterRestartIT.java | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java new file mode 100644 index 0000000000000..15f420b2cab55 --- /dev/null +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.restart; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; +import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class MlHiddenIndicesFullClusterRestartIT extends AbstractFullClusterRestartTestCase { + + private static final String OLD_CLUSTER_JOB_ID = "ml-hidden-indices-old-cluster-job"; + + @Override + protected Settings restClientSettings() { + String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); + } + + @Before + public void waitForMlTemplates() throws Exception { + List templatesToWaitFor = (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_12_0)) + ? XPackRestTestConstants.ML_POST_V660_TEMPLATES + : XPackRestTestConstants.ML_POST_V7120_TEMPLATES; + boolean clusterUnderstandsComposableTemplates = + isRunningAgainstOldCluster() == false || getOldClusterVersion().onOrAfter(Version.V_7_8_0); + XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor, clusterUnderstandsComposableTemplates); + } + + public void testMlIndicesBecomeHidden() throws Exception { + if (isRunningAgainstOldCluster()) { + assertThatNoMlIndicesExist(); + // trigger ML indices creation + createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); + + if (getOldClusterVersion().before(Version.V_7_7_0)) { + Map indexSettings = contentAsMap(getMlIndicesSettings()); + for (Map.Entry e : indexSettings.entrySet()) { + String indexName = e.getKey(); + @SuppressWarnings("unchecked") + Map settings = (Map) e.getValue(); + assertThat(settings, is(notNullValue())); + assertThat("Index " + indexName + " expected not to be hidden but was", + XContentMapValues.extractValue(settings, "settings", "index", "hidden"), + is(nullValue())); + } + } + } else { + Map indexSettings = contentAsMap(getMlIndicesSettings()); + for (Map.Entry e : indexSettings.entrySet()) { + String indexName = e.getKey(); + // .ml-config is supposed to be a system index, *not* a hidden index + if (".ml-config".equals(indexName)) { + continue; + } + @SuppressWarnings("unchecked") + Map settings = (Map) e.getValue(); + assertThat(settings, is(notNullValue())); + assertThat("Index " + indexName + " expected to be hidden but wasn't", + XContentMapValues.extractValue(settings, "settings", "index", "hidden"), + is(equalTo("true"))); + } + } + } + + private Response getMlIndicesSettings() throws IOException { + Request getSettingsRequest = new Request("GET", ".ml-*/_settings"); + getSettingsRequest.setOptions(expectVersionSpecificWarnings(v -> { + final String systemIndexWarning = "this request accesses system indices: [.ml-config], but in a future major version, direct " + + "access to system indices will be prevented by default"; + v.current(systemIndexWarning); + v.compatible(systemIndexWarning); + })); + Response getSettingsResponse = client().performRequest(getSettingsRequest); + assertThat(getSettingsResponse, is(notNullValue())); + return getSettingsResponse; + } + + @SuppressWarnings("unchecked") + private static Map contentAsMap(Response response) throws IOException { + return new ObjectMapper().readValue(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8), HashMap.class); + } + + private void assertThatNoMlIndicesExist() throws IOException { + Request getIndexRequest = new Request("GET", ".ml-*"); + getIndexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addParameter("allow_no_indices", "false").build()); + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(getIndexRequest)); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + } + + private void createAnomalyDetectorJob(String jobId) throws IOException { + String jobConfig = + "{\n" + + " \"job_id\": \"" + jobId + "\",\n" + + " \"analysis_config\": {\n" + + " \"bucket_span\": \"10m\",\n" + + " \"detectors\": [{\n" + + " \"function\": \"metric\",\n" + + " \"field_name\": \"responsetime\"\n" + + " }]\n" + + " },\n" + + " \"data_description\": {}\n" + + "}"; + + Request putJobRequest = new Request("PUT", "/_ml/anomaly_detectors/" + jobId); + putJobRequest.setJsonEntity(jobConfig); + Response putJobResponse = client().performRequest(putJobRequest); + assertThat(putJobResponse.getStatusLine().getStatusCode(), equalTo(200)); + } +} From 6bb53eaa2ebfe7840e37ad77666e4f79aa5bf638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 21 Sep 2021 09:29:58 +0200 Subject: [PATCH 05/18] Apply review comments --- .../java/org/elasticsearch/xpack/ml/MachineLearning.java | 4 ++++ .../elasticsearch/xpack/ml/MlInitializationService.java | 7 ++++--- .../xpack/restart/MlHiddenIndicesFullClusterRestartIT.java | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 8bcce63145988..b029d20690be5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1419,6 +1419,10 @@ public static SystemIndexDescriptor getInferenceIndexSecurityDescriptor() { .build(); } + /** + * These are the ML hidden indices. They are "associated" in the sense that if the ML system indices + * are backed up or deleted then these hidden indices should also be backed up or deleted. + */ private static Collection ASSOCIATED_INDEX_DESCRIPTORS = List.of( new AssociatedIndexDescriptor(RESULTS_INDEX_PREFIX + "*", "Results indices"), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 4da7f97dd8776..c6336fb8414f2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -139,11 +139,11 @@ private void makeMlIndicesHidden() { .map(Map.Entry::getKey) .toArray(String[]::new); if (nonHiddenIndices.length == 0) { - logger.info("There are no indices that need to be made hidden, " + getSettingsResponse); + logger.debug("There are no ML internal indices that need to be made hidden, " + getSettingsResponse); return; } String nonHiddenIndicesString = Arrays.stream(nonHiddenIndices).collect(Collectors.joining(", ")); - logger.info("The following indices will now be made hidden: {}", nonHiddenIndicesString); + logger.info("The following ML internal indices will now be made hidden: {}", nonHiddenIndicesString); AcknowledgedResponse updateSettingsResponse = client.admin().indices().prepareUpdateSettings() .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) @@ -151,8 +151,9 @@ private void makeMlIndicesHidden() { .setSettings(Collections.singletonMap(SETTING_INDEX_HIDDEN, true)) .get(); if (updateSettingsResponse.isAcknowledged() == false) { - logger.error("One or more of the following indices could not be made hidden: {}", nonHiddenIndicesString); + logger.error("One or more of the following ML internal indices could not be made hidden: {}", nonHiddenIndicesString); } + // TODO: Make aliases hidden } } diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 15f420b2cab55..b1b2b99e9e725 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -107,7 +107,8 @@ private Response getMlIndicesSettings() throws IOException { @SuppressWarnings("unchecked") private static Map contentAsMap(Response response) throws IOException { - return new ObjectMapper().readValue(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8), HashMap.class); + return new ObjectMapper().readValue( + new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8), HashMap.class); } private void assertThatNoMlIndicesExist() throws IOException { From 6cfe8b28d05a947d1968c4c57c98b590b436ded9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Mon, 4 Oct 2021 12:59:00 +0200 Subject: [PATCH 06/18] Make ML aliases hidden --- .../xpack/ml/MlInitializationService.java | 145 +++++++++++++++--- .../ml/MlInitializationServiceTests.java | 6 +- .../MlHiddenIndicesFullClusterRestartIT.java | 59 ++++++- 3 files changed, 181 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index c6336fb8414f2..8281352c71e4a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -6,16 +6,28 @@ */ package org.elasticsearch.xpack.ml; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.LatchedActionListener; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.settings.Settings; @@ -25,12 +37,16 @@ import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; public class MlInitializationService implements ClusterStateListener { @@ -128,32 +144,111 @@ MlDailyMaintenanceService getDailyMaintenanceService() { private void makeMlIndicesHidden() { String[] mlHiddenIndexPatterns = MachineLearning.getMlHiddenIndexPatterns(); - GetSettingsResponse getSettingsResponse = - client.admin().indices().prepareGetSettings() - .setIndices(mlHiddenIndexPatterns) - .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) - .get(); - String[] nonHiddenIndices = - getSettingsResponse.getIndexToSettings().stream() - .filter(e -> e.getValue().getAsBoolean(SETTING_INDEX_HIDDEN, false) == false) - .map(Map.Entry::getKey) - .toArray(String[]::new); - if (nonHiddenIndices.length == 0) { - logger.debug("There are no ML internal indices that need to be made hidden, " + getSettingsResponse); - return; - } - String nonHiddenIndicesString = Arrays.stream(nonHiddenIndices).collect(Collectors.joining(", ")); - logger.info("The following ML internal indices will now be made hidden: {}", nonHiddenIndicesString); - AcknowledgedResponse updateSettingsResponse = - client.admin().indices().prepareUpdateSettings() - .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) - .setIndices(nonHiddenIndices) - .setSettings(Collections.singletonMap(SETTING_INDEX_HIDDEN, true)) - .get(); - if (updateSettingsResponse.isAcknowledged() == false) { - logger.error("One or more of the following ML internal indices could not be made hidden: {}", nonHiddenIndicesString); + + CountDownLatch latch = new CountDownLatch(1); + + LatchedActionListener listener = new LatchedActionListener<>( + ActionListener.wrap( + result -> logger.info("All the ML internal indices and aliases were successfully made hidden"), + e -> logger.error("An error occurred while making ML indices and aliases hidden", e) + ), + latch + ); + + ActionListener updateAliasesListener = ActionListener.wrap( + updateAliasesResponse -> { + if (updateAliasesResponse.isAcknowledged() == false) { + logger.error("One or more of the following ML aliases could not be made hidden: {}", "sraka"); + } + listener.onResponse(updateAliasesResponse.isAcknowledged()); + }, + listener::onFailure + ); + + ActionListener getAliasesResponseListener = ActionListener.wrap( + getAliasesResponse -> { + IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); + for (ObjectObjectCursor> entry : getAliasesResponse.getAliases()) { + String index = entry.key; + String[] nonHiddenAliases = entry.value.stream() + .filter(metadata -> metadata.isHidden() == null || metadata.isHidden() == false) + .map(AliasMetadata::alias) + .toArray(String[]::new); + if (nonHiddenAliases.length == 0) { + continue; + } + indicesAliasesRequest.addAliasAction( + IndicesAliasesRequest.AliasActions.add() + .index(index) + .aliases(entry.value.stream().map(AliasMetadata::alias).toArray(String[]::new)) + .isHidden(true)); + } + if (indicesAliasesRequest.getAliasActions().isEmpty()) { + logger.debug("There are no ML internal aliases that need to be made hidden, " + getAliasesResponse.getAliases()); + listener.onResponse(true); + return; + } + String indicesWithNonHiddenAliasesString = + indicesAliasesRequest.getAliasActions().stream() + .map(aa -> aa.indices()[0] + "/" + String.join(",", aa.aliases())) + .collect(Collectors.joining(", ")); + logger.debug("The following ML internal aliases will now be made hidden: [{}]", indicesWithNonHiddenAliasesString); + executeAsyncWithOrigin(client, ML_ORIGIN, IndicesAliasesAction.INSTANCE, indicesAliasesRequest, updateAliasesListener); + }, + listener::onFailure + ); + + ActionListener updateSettingsListener = ActionListener.wrap( + updateSettingsResponse -> { + if (updateSettingsResponse.isAcknowledged() == false) { + logger.error("One or more of the ML internal indices could not be made hidden."); + listener.onResponse(false); + return; + } + GetAliasesRequest getAliasesRequest = new GetAliasesRequest() + .indices(mlHiddenIndexPatterns) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN); + executeAsyncWithOrigin(client, ML_ORIGIN, GetAliasesAction.INSTANCE, getAliasesRequest, getAliasesResponseListener); + }, + listener::onFailure + ); + + ActionListener getSettingsListener = ActionListener.wrap( + getSettingsResponse -> { + String[] nonHiddenIndices = + getSettingsResponse.getIndexToSettings().stream() + .filter(e -> e.getValue().getAsBoolean(SETTING_INDEX_HIDDEN, false) == false) + .map(Map.Entry::getKey) + .toArray(String[]::new); + if (nonHiddenIndices.length == 0) { + logger.debug("There are no ML internal indices that need to be made hidden, " + getSettingsResponse); + listener.onResponse(true); + return; + } + String nonHiddenIndicesString = Arrays.stream(nonHiddenIndices).collect(Collectors.joining(", ")); + logger.debug("The following ML internal indices will now be made hidden: [{}]", nonHiddenIndicesString); + UpdateSettingsRequest updateSettingsRequest = + new UpdateSettingsRequest() + .indices(nonHiddenIndices) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN) + .settings(Collections.singletonMap(SETTING_INDEX_HIDDEN, true)); + executeAsyncWithOrigin(client, ML_ORIGIN, UpdateSettingsAction.INSTANCE, updateSettingsRequest, updateSettingsListener); + }, + listener::onFailure + ); + + GetSettingsRequest getSettingsRequest = + new GetSettingsRequest() + .indices(mlHiddenIndexPatterns) + .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN); + getSettingsListener.onResponse(client.admin().indices().getSettings(getSettingsRequest).actionGet()); +// executeAsyncWithOrigin(client, ML_ORIGIN, GetSettingsAction.INSTANCE, getSettingsRequest, getSettingsListener); + + try { + latch.await(); + } catch (InterruptedException e) { + logger.error("Interrupted while making ML indices and aliases hidden", e); } - // TODO: Make aliases hidden } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java index 9f4700117b2d1..10c2a2dfc45b5 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlInitializationServiceTests.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; @@ -62,8 +61,11 @@ public void setUpMocks() { when(clusterService.getClusterName()).thenReturn(CLUSTER_NAME); + @SuppressWarnings("unchecked") + ActionFuture getSettingsResponseActionFuture = mock(ActionFuture.class); + when(getSettingsResponseActionFuture.actionGet()).thenReturn(new GetSettingsResponse(ImmutableOpenMap.of(), ImmutableOpenMap.of())); IndicesAdminClient indicesAdminClient = mock(IndicesAdminClient.class); - when(indicesAdminClient.prepareGetSettings()).thenReturn(new GetSettingsRequestBuilder(client, GetSettingsAction.INSTANCE)); + when(indicesAdminClient.getSettings(any())).thenReturn(getSettingsResponseActionFuture); AdminClient adminClient = mock(AdminClient.class); when(adminClient.indices()).thenReturn(indicesAdminClient); when(client.admin()).thenReturn(adminClient); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index b1b2b99e9e725..28453f2c158a0 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -16,7 +16,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.Tuple; import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; +import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; import org.junit.Before; @@ -24,6 +28,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.Base64; import java.util.HashMap; import java.util.List; @@ -61,6 +66,26 @@ public void testMlIndicesBecomeHidden() throws Exception { assertThatNoMlIndicesExist(); // trigger ML indices creation createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); + openAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); + + Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); + getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { + v.current(systemAliasWarning); + v.compatible(systemAliasWarning); + })); + Response getAliasesResponse = client().performRequest(getAliasesRequest); + Map aliasesMap = contentAsMap(getAliasesResponse); + List> expected = + List.of( + Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.READ_ALIAS_NAME), + Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.WRITE_ALIAS_NAME), + Tuple.tuple(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobStateIndexWriteAlias())); + for (Tuple indexAndAlias : expected) { + assertThat( + indexAndAlias + " should not be hidden", + XContentMapValues.extractValue(aliasesMap, indexAndAlias.v1(), "aliases", indexAndAlias.v2(), "is_hidden"), + is(nullValue())); + } if (getOldClusterVersion().before(Version.V_7_7_0)) { Map indexSettings = contentAsMap(getMlIndicesSettings()); @@ -89,14 +114,38 @@ public void testMlIndicesBecomeHidden() throws Exception { XContentMapValues.extractValue(settings, "settings", "index", "hidden"), is(equalTo("true"))); } + + Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); + getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { + v.current(systemAliasWarning); + v.compatible(systemAliasWarning); + })); + Response getAliasesResponse = client().performRequest(getAliasesRequest); + Map aliasesMap = contentAsMap(getAliasesResponse); + // TODO: Include all the ML aliases + List> expected = + List.of( + Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.READ_ALIAS_NAME), + Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.WRITE_ALIAS_NAME) + ); + for (Tuple indexAndAlias : expected) { + assertThat( + indexAndAlias + " should be hidden, " + aliasesMap, + XContentMapValues.extractValue(aliasesMap, indexAndAlias.v1(), "aliases", indexAndAlias.v2(), "is_hidden"), + is(true)); + } } } + private static final String systemIndexWarning = "this request accesses system indices: [.ml-config], but in a future major version, direct " + + "access to system indices will be prevented by default"; + + private static final String systemAliasWarning = "this request accesses aliases with names reserved for system indices: [.security], " + + "but in a future major version, direct access to system indices and their aliases will not be allowed"; + private Response getMlIndicesSettings() throws IOException { Request getSettingsRequest = new Request("GET", ".ml-*/_settings"); getSettingsRequest.setOptions(expectVersionSpecificWarnings(v -> { - final String systemIndexWarning = "this request accesses system indices: [.ml-config], but in a future major version, direct " + - "access to system indices will be prevented by default"; v.current(systemIndexWarning); v.compatible(systemIndexWarning); })); @@ -137,4 +186,10 @@ private void createAnomalyDetectorJob(String jobId) throws IOException { Response putJobResponse = client().performRequest(putJobRequest); assertThat(putJobResponse.getStatusLine().getStatusCode(), equalTo(200)); } + + private void openAnomalyDetectorJob(String jobId) throws IOException { + Request openJobRequest = new Request("POST", "/_ml/anomaly_detectors/" + jobId + "/_open"); + Response openJobResponse = client().performRequest(openJobRequest); + assertThat(openJobResponse.getStatusLine().getStatusCode(), equalTo(200)); + } } From 17e72e99c1f8b15817fefbabb667063c436709c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Mon, 4 Oct 2021 13:41:02 +0200 Subject: [PATCH 07/18] Expect warnings related to system indices --- .../restart/MlHiddenIndicesFullClusterRestartIT.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 28453f2c158a0..28e66738e4a6c 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -79,7 +79,8 @@ public void testMlIndicesBecomeHidden() throws Exception { List.of( Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.READ_ALIAS_NAME), Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.WRITE_ALIAS_NAME), - Tuple.tuple(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobStateIndexWriteAlias())); + Tuple.tuple(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobStateIndexWriteAlias()) + ); for (Tuple indexAndAlias : expected) { assertThat( indexAndAlias + " should not be hidden", @@ -162,7 +163,12 @@ private static Map contentAsMap(Response response) throws IOExce private void assertThatNoMlIndicesExist() throws IOException { Request getIndexRequest = new Request("GET", ".ml-*"); - getIndexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addParameter("allow_no_indices", "false").build()); + getIndexRequest.setOptions(expectVersionSpecificWarnings(v -> { + v.current(systemIndexWarning); + v.compatible(systemIndexWarning); + }).toBuilder() + .addParameter("allow_no_indices", "false") + .build()); ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(getIndexRequest)); assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); } From 461fdac0353dc60423f726b61548369d11e5c4b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Mon, 4 Oct 2021 13:43:26 +0200 Subject: [PATCH 08/18] Fix style --- .../restart/MlHiddenIndicesFullClusterRestartIT.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 28e66738e4a6c..17b92d35bcba7 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -10,7 +10,6 @@ import org.elasticsearch.Version; import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.Settings; @@ -28,7 +27,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.time.Instant; import java.util.Base64; import java.util.HashMap; import java.util.List; @@ -68,7 +66,8 @@ public void testMlIndicesBecomeHidden() throws Exception { createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); openAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); - Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); + Request getAliasesRequest = + new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { v.current(systemAliasWarning); v.compatible(systemAliasWarning); @@ -116,7 +115,8 @@ public void testMlIndicesBecomeHidden() throws Exception { is(equalTo("true"))); } - Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); + Request getAliasesRequest = + new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { v.current(systemAliasWarning); v.compatible(systemAliasWarning); @@ -138,8 +138,8 @@ public void testMlIndicesBecomeHidden() throws Exception { } } - private static final String systemIndexWarning = "this request accesses system indices: [.ml-config], but in a future major version, direct " + - "access to system indices will be prevented by default"; + private static final String systemIndexWarning = "this request accesses system indices: [.ml-config], but in a future major version, " + + "direct access to system indices will be prevented by default"; private static final String systemAliasWarning = "this request accesses aliases with names reserved for system indices: [.security], " + "but in a future major version, direct access to system indices and their aliases will not be allowed"; From 60edef730c653ed14a87ce1c791e069c77b125aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Mon, 4 Oct 2021 16:12:00 +0200 Subject: [PATCH 09/18] Fix integration and BWC tests --- .../MlInitializationServiceIT.java | 8 ++++++ .../MlHiddenIndicesFullClusterRestartIT.java | 26 +++++++++---------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java index 32e886f404602..9caa1d56e9e8c 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -104,4 +105,11 @@ public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() { settings.getAsBoolean(SETTING_INDEX_HIDDEN, false), is(equalTo(false))); } } + + @Override + public Settings indexSettings() { + return Settings.builder().put(super.indexSettings()) + .put(IndexMetadata.SETTING_DATA_PATH, (String) null) + .build(); + } } diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 17b92d35bcba7..9ef8bed05330d 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -74,19 +74,6 @@ public void testMlIndicesBecomeHidden() throws Exception { })); Response getAliasesResponse = client().performRequest(getAliasesRequest); Map aliasesMap = contentAsMap(getAliasesResponse); - List> expected = - List.of( - Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.READ_ALIAS_NAME), - Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.WRITE_ALIAS_NAME), - Tuple.tuple(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobStateIndexWriteAlias()) - ); - for (Tuple indexAndAlias : expected) { - assertThat( - indexAndAlias + " should not be hidden", - XContentMapValues.extractValue(aliasesMap, indexAndAlias.v1(), "aliases", indexAndAlias.v2(), "is_hidden"), - is(nullValue())); - } - if (getOldClusterVersion().before(Version.V_7_7_0)) { Map indexSettings = contentAsMap(getMlIndicesSettings()); for (Map.Entry e : indexSettings.entrySet()) { @@ -98,6 +85,19 @@ public void testMlIndicesBecomeHidden() throws Exception { XContentMapValues.extractValue(settings, "settings", "index", "hidden"), is(nullValue())); } + + List> expected = + List.of( + Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.READ_ALIAS_NAME), + Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.WRITE_ALIAS_NAME), + Tuple.tuple(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobStateIndexWriteAlias()) + ); + for (Tuple indexAndAlias : expected) { + assertThat( + indexAndAlias + " should not be hidden", + XContentMapValues.extractValue(aliasesMap, indexAndAlias.v1(), "aliases", indexAndAlias.v2(), "is_hidden"), + is(nullValue())); + } } } else { Map indexSettings = contentAsMap(getMlIndicesSettings()); From 61b881c330244cd61bb566c526c448c4b856dab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 5 Oct 2021 08:00:04 +0200 Subject: [PATCH 10/18] Remove assertion which may not be true if many tests are running at once. --- .../MlHiddenIndicesFullClusterRestartIT.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 9ef8bed05330d..c26dc2355c2df 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -11,7 +11,6 @@ import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -61,7 +60,6 @@ public void waitForMlTemplates() throws Exception { public void testMlIndicesBecomeHidden() throws Exception { if (isRunningAgainstOldCluster()) { - assertThatNoMlIndicesExist(); // trigger ML indices creation createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); openAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); @@ -161,18 +159,6 @@ private static Map contentAsMap(Response response) throws IOExce new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8), HashMap.class); } - private void assertThatNoMlIndicesExist() throws IOException { - Request getIndexRequest = new Request("GET", ".ml-*"); - getIndexRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemIndexWarning); - v.compatible(systemIndexWarning); - }).toBuilder() - .addParameter("allow_no_indices", "false") - .build()); - ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(getIndexRequest)); - assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); - } - private void createAnomalyDetectorJob(String jobId) throws IOException { String jobConfig = "{\n" + From e2b672ef787022e65ed1a5184a417314918855d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 5 Oct 2021 09:26:18 +0200 Subject: [PATCH 11/18] Extract common code to a method --- .../MlHiddenIndicesFullClusterRestartIT.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index c26dc2355c2df..4808c8f05446a 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -64,17 +64,11 @@ public void testMlIndicesBecomeHidden() throws Exception { createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); openAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); - Request getAliasesRequest = - new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); - getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemAliasWarning); - v.compatible(systemAliasWarning); - })); - Response getAliasesResponse = client().performRequest(getAliasesRequest); - Map aliasesMap = contentAsMap(getAliasesResponse); if (getOldClusterVersion().before(Version.V_7_7_0)) { - Map indexSettings = contentAsMap(getMlIndicesSettings()); - for (Map.Entry e : indexSettings.entrySet()) { + Map indexSettingsMap = contentAsMap(getMlIndicesSettings()); + Map aliasesMap = contentAsMap(getMlAliases()); + + for (Map.Entry e : indexSettingsMap.entrySet()) { String indexName = e.getKey(); @SuppressWarnings("unchecked") Map settings = (Map) e.getValue(); @@ -98,8 +92,10 @@ public void testMlIndicesBecomeHidden() throws Exception { } } } else { - Map indexSettings = contentAsMap(getMlIndicesSettings()); - for (Map.Entry e : indexSettings.entrySet()) { + Map indexSettingsMap = contentAsMap(getMlIndicesSettings()); + Map aliasesMap = contentAsMap(getMlAliases()); + + for (Map.Entry e : indexSettingsMap.entrySet()) { String indexName = e.getKey(); // .ml-config is supposed to be a system index, *not* a hidden index if (".ml-config".equals(indexName)) { @@ -113,14 +109,6 @@ public void testMlIndicesBecomeHidden() throws Exception { is(equalTo("true"))); } - Request getAliasesRequest = - new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); - getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemAliasWarning); - v.compatible(systemAliasWarning); - })); - Response getAliasesResponse = client().performRequest(getAliasesRequest); - Map aliasesMap = contentAsMap(getAliasesResponse); // TODO: Include all the ML aliases List> expected = List.of( @@ -139,9 +127,6 @@ public void testMlIndicesBecomeHidden() throws Exception { private static final String systemIndexWarning = "this request accesses system indices: [.ml-config], but in a future major version, " + "direct access to system indices will be prevented by default"; - private static final String systemAliasWarning = "this request accesses aliases with names reserved for system indices: [.security], " + - "but in a future major version, direct access to system indices and their aliases will not be allowed"; - private Response getMlIndicesSettings() throws IOException { Request getSettingsRequest = new Request("GET", ".ml-*/_settings"); getSettingsRequest.setOptions(expectVersionSpecificWarnings(v -> { @@ -153,6 +138,21 @@ private Response getMlIndicesSettings() throws IOException { return getSettingsResponse; } + private static final String systemAliasWarning = "this request accesses aliases with names reserved for system indices: [.security], " + + "but in a future major version, direct access to system indices and their aliases will not be allowed"; + + private Response getMlAliases() throws IOException { + Request getAliasesRequest = + new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); + getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { + v.current(systemAliasWarning); + v.compatible(systemAliasWarning); + })); + Response getAliasesResponse = client().performRequest(getAliasesRequest); + assertThat(getAliasesResponse, is(notNullValue())); + return getAliasesResponse; + } + @SuppressWarnings("unchecked") private static Map contentAsMap(Response response) throws IOException { return new ObjectMapper().readValue( From bee3040dcffa13d803157e775e4d1321ce304d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 5 Oct 2021 09:58:23 +0200 Subject: [PATCH 12/18] Adjust log messages --- .../elasticsearch/xpack/ml/MlInitializationService.java | 6 +++--- .../xpack/restart/MlHiddenIndicesFullClusterRestartIT.java | 7 ------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 8281352c71e4a..f50326ea5a56b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -150,7 +150,7 @@ private void makeMlIndicesHidden() { LatchedActionListener listener = new LatchedActionListener<>( ActionListener.wrap( result -> logger.info("All the ML internal indices and aliases were successfully made hidden"), - e -> logger.error("An error occurred while making ML indices and aliases hidden", e) + e -> logger.error("An error occurred while making ML internal indices and aliases hidden", e) ), latch ); @@ -158,7 +158,7 @@ private void makeMlIndicesHidden() { ActionListener updateAliasesListener = ActionListener.wrap( updateAliasesResponse -> { if (updateAliasesResponse.isAcknowledged() == false) { - logger.error("One or more of the following ML aliases could not be made hidden: {}", "sraka"); + logger.error("One or more of the following ML internal aliases could not be made hidden."); } listener.onResponse(updateAliasesResponse.isAcknowledged()); }, @@ -247,7 +247,7 @@ private void makeMlIndicesHidden() { try { latch.await(); } catch (InterruptedException e) { - logger.error("Interrupted while making ML indices and aliases hidden", e); + logger.error("Interrupted while making ML internal indices and aliases hidden", e); } } } diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 4808c8f05446a..9e7805e268bab 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -138,16 +138,9 @@ private Response getMlIndicesSettings() throws IOException { return getSettingsResponse; } - private static final String systemAliasWarning = "this request accesses aliases with names reserved for system indices: [.security], " + - "but in a future major version, direct access to system indices and their aliases will not be allowed"; - private Response getMlAliases() throws IOException { Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); - getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemAliasWarning); - v.compatible(systemAliasWarning); - })); Response getAliasesResponse = client().performRequest(getAliasesRequest); assertThat(getAliasesResponse, is(notNullValue())); return getAliasesResponse; From 4596f6093e7be392997124cf5fd546a4d65fecdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 5 Oct 2021 10:29:32 +0200 Subject: [PATCH 13/18] Restore warning expectation --- .../xpack/restart/MlHiddenIndicesFullClusterRestartIT.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 9e7805e268bab..4808c8f05446a 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -138,9 +138,16 @@ private Response getMlIndicesSettings() throws IOException { return getSettingsResponse; } + private static final String systemAliasWarning = "this request accesses aliases with names reserved for system indices: [.security], " + + "but in a future major version, direct access to system indices and their aliases will not be allowed"; + private Response getMlAliases() throws IOException { Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); + getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { + v.current(systemAliasWarning); + v.compatible(systemAliasWarning); + })); Response getAliasesResponse = client().performRequest(getAliasesRequest); assertThat(getAliasesResponse, is(notNullValue())); return getAliasesResponse; From 36d5069b09b1a09d37dfe2392833a8226eebf93f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 5 Oct 2021 10:31:11 +0200 Subject: [PATCH 14/18] Do not require warning in the mixed-version environment --- .../restart/MlHiddenIndicesFullClusterRestartIT.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 4808c8f05446a..9fab8847b487a 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -129,10 +129,7 @@ public void testMlIndicesBecomeHidden() throws Exception { private Response getMlIndicesSettings() throws IOException { Request getSettingsRequest = new Request("GET", ".ml-*/_settings"); - getSettingsRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemIndexWarning); - v.compatible(systemIndexWarning); - })); + getSettingsRequest.setOptions(expectVersionSpecificWarnings(v -> v.compatible(systemIndexWarning))); Response getSettingsResponse = client().performRequest(getSettingsRequest); assertThat(getSettingsResponse, is(notNullValue())); return getSettingsResponse; @@ -144,10 +141,7 @@ private Response getMlIndicesSettings() throws IOException { private Response getMlAliases() throws IOException { Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); - getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemAliasWarning); - v.compatible(systemAliasWarning); - })); + getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> v.compatible(systemAliasWarning))); Response getAliasesResponse = client().performRequest(getAliasesRequest); assertThat(getAliasesResponse, is(notNullValue())); return getAliasesResponse; From 084062547dfd794243d8ae48feecfcd88d5d78d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 5 Oct 2021 16:25:22 +0200 Subject: [PATCH 15/18] Ignore REST warnings --- .../MlHiddenIndicesFullClusterRestartIT.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 9fab8847b487a..e48ffa4015ed7 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -10,7 +10,9 @@ import org.elasticsearch.Version; import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; +import org.elasticsearch.client.WarningsHandler; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; @@ -124,24 +126,24 @@ public void testMlIndicesBecomeHidden() throws Exception { } } - private static final String systemIndexWarning = "this request accesses system indices: [.ml-config], but in a future major version, " + - "direct access to system indices will be prevented by default"; - private Response getMlIndicesSettings() throws IOException { Request getSettingsRequest = new Request("GET", ".ml-*/_settings"); - getSettingsRequest.setOptions(expectVersionSpecificWarnings(v -> v.compatible(systemIndexWarning))); + getSettingsRequest + .setOptions(RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(WarningsHandler.PERMISSIVE) + .build()); Response getSettingsResponse = client().performRequest(getSettingsRequest); assertThat(getSettingsResponse, is(notNullValue())); return getSettingsResponse; } - private static final String systemAliasWarning = "this request accesses aliases with names reserved for system indices: [.security], " + - "but in a future major version, direct access to system indices and their aliases will not be allowed"; - private Response getMlAliases() throws IOException { Request getAliasesRequest = new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_alias"); - getAliasesRequest.setOptions(expectVersionSpecificWarnings(v -> v.compatible(systemAliasWarning))); + getAliasesRequest + .setOptions(RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(WarningsHandler.PERMISSIVE) + .build()); Response getAliasesResponse = client().performRequest(getAliasesRequest); assertThat(getAliasesResponse, is(notNullValue())); return getAliasesResponse; From fe9c776c3cf5c0bbe8dfddc3de0203653db53d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Thu, 7 Oct 2021 10:16:15 +0200 Subject: [PATCH 16/18] Apply review comments --- .../xpack/ml/MachineLearning.java | 2 +- .../xpack/ml/MlInitializationService.java | 55 +++++++------------ .../MlHiddenIndicesFullClusterRestartIT.java | 52 ++++++++---------- 3 files changed, 44 insertions(+), 65 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index b029d20690be5..0ab1a628ea285 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1429,7 +1429,7 @@ public static SystemIndexDescriptor getInferenceIndexSecurityDescriptor() { new AssociatedIndexDescriptor(STATE_INDEX_PREFIX + "*", "State indices"), new AssociatedIndexDescriptor(MlStatsIndex.indexPattern(), "ML stats index"), new AssociatedIndexDescriptor(".ml-notifications*", "ML notifications indices"), - new AssociatedIndexDescriptor(".ml-annotations*", "Ml annotations indices") + new AssociatedIndexDescriptor(".ml-annotations*", "ML annotations indices") ); @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index f50326ea5a56b..87b92bd36d9b1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; @@ -40,7 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -145,26 +143,19 @@ MlDailyMaintenanceService getDailyMaintenanceService() { private void makeMlIndicesHidden() { String[] mlHiddenIndexPatterns = MachineLearning.getMlHiddenIndexPatterns(); - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener listener = new LatchedActionListener<>( - ActionListener.wrap( - result -> logger.info("All the ML internal indices and aliases were successfully made hidden"), - e -> logger.error("An error occurred while making ML internal indices and aliases hidden", e) - ), - latch - ); - - ActionListener updateAliasesListener = ActionListener.wrap( + // Step 5: Handle errors encountered on the way. + ActionListener finalListener = ActionListener.wrap( updateAliasesResponse -> { if (updateAliasesResponse.isAcknowledged() == false) { - logger.error("One or more of the following ML internal aliases could not be made hidden."); + logger.error("One or more of the ML internal aliases could not be made hidden."); + return; } - listener.onResponse(updateAliasesResponse.isAcknowledged()); + logger.debug("All the ML internal indices and aliases were successfully made hidden"); }, - listener::onFailure + e -> logger.error("An error occurred while making ML internal indices and aliases hidden", e) ); + // Step 4: Extract ML internal aliases that are not hidden and make them hidden. ActionListener getAliasesResponseListener = ActionListener.wrap( getAliasesResponse -> { IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); @@ -184,25 +175,24 @@ private void makeMlIndicesHidden() { .isHidden(true)); } if (indicesAliasesRequest.getAliasActions().isEmpty()) { - logger.debug("There are no ML internal aliases that need to be made hidden, " + getAliasesResponse.getAliases()); - listener.onResponse(true); + logger.debug("There are no ML internal aliases that need to be made hidden, [{}]", getAliasesResponse.getAliases()); return; } String indicesWithNonHiddenAliasesString = indicesAliasesRequest.getAliasActions().stream() - .map(aa -> aa.indices()[0] + "/" + String.join(",", aa.aliases())) - .collect(Collectors.joining(", ")); + .map(aliasAction -> aliasAction.indices()[0] + ": " + String.join(",", aliasAction.aliases())) + .collect(Collectors.joining("; ")); logger.debug("The following ML internal aliases will now be made hidden: [{}]", indicesWithNonHiddenAliasesString); - executeAsyncWithOrigin(client, ML_ORIGIN, IndicesAliasesAction.INSTANCE, indicesAliasesRequest, updateAliasesListener); + executeAsyncWithOrigin(client, ML_ORIGIN, IndicesAliasesAction.INSTANCE, indicesAliasesRequest, finalListener); }, - listener::onFailure + finalListener::onFailure ); + // Step 3: Once indices are hidden, fetch ML internal aliases to find out whether the aliases are hidden or not. ActionListener updateSettingsListener = ActionListener.wrap( updateSettingsResponse -> { if (updateSettingsResponse.isAcknowledged() == false) { logger.error("One or more of the ML internal indices could not be made hidden."); - listener.onResponse(false); return; } GetAliasesRequest getAliasesRequest = new GetAliasesRequest() @@ -210,9 +200,10 @@ private void makeMlIndicesHidden() { .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN); executeAsyncWithOrigin(client, ML_ORIGIN, GetAliasesAction.INSTANCE, getAliasesRequest, getAliasesResponseListener); }, - listener::onFailure + finalListener::onFailure ); + // Step 2: Extract ML internal indices that are not hidden and make them hidden. ActionListener getSettingsListener = ActionListener.wrap( getSettingsResponse -> { String[] nonHiddenIndices = @@ -221,8 +212,8 @@ private void makeMlIndicesHidden() { .map(Map.Entry::getKey) .toArray(String[]::new); if (nonHiddenIndices.length == 0) { - logger.debug("There are no ML internal indices that need to be made hidden, " + getSettingsResponse); - listener.onResponse(true); + logger.debug("There are no ML internal indices that need to be made hidden, [{}]", getSettingsResponse); + updateSettingsListener.onResponse(AcknowledgedResponse.TRUE); return; } String nonHiddenIndicesString = Arrays.stream(nonHiddenIndices).collect(Collectors.joining(", ")); @@ -234,21 +225,15 @@ private void makeMlIndicesHidden() { .settings(Collections.singletonMap(SETTING_INDEX_HIDDEN, true)); executeAsyncWithOrigin(client, ML_ORIGIN, UpdateSettingsAction.INSTANCE, updateSettingsRequest, updateSettingsListener); }, - listener::onFailure + finalListener::onFailure ); + // Step 1: Fetch ML internal indices settings to find out whether they are already hidden or not. GetSettingsRequest getSettingsRequest = new GetSettingsRequest() .indices(mlHiddenIndexPatterns) .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN); - getSettingsListener.onResponse(client.admin().indices().getSettings(getSettingsRequest).actionGet()); -// executeAsyncWithOrigin(client, ML_ORIGIN, GetSettingsAction.INSTANCE, getSettingsRequest, getSettingsListener); - - try { - latch.await(); - } catch (InterruptedException e) { - logger.error("Interrupted while making ML internal indices and aliases hidden", e); - } + client.admin().indices().getSettings(getSettingsRequest, getSettingsListener); } } diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index e48ffa4015ed7..3bdd3ac7eea99 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -18,9 +18,6 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Tuple; import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase; -import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; import org.junit.Before; @@ -33,14 +30,24 @@ import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class MlHiddenIndicesFullClusterRestartIT extends AbstractFullClusterRestartTestCase { - private static final String OLD_CLUSTER_JOB_ID = "ml-hidden-indices-old-cluster-job"; + private static final String JOB_ID = "ml-hidden-indices-old-cluster-job"; + private static final List> EXPECTED_INDEX_ALIAS_PAIRS = + List.of( + Tuple.tuple(".ml-annotations-6", ".ml-annotations-read"), + Tuple.tuple(".ml-annotations-6", ".ml-annotations-write"), + Tuple.tuple(".ml-state-000001", ".ml-state-write"), + Tuple.tuple(".ml-anomalies-shared", ".ml-anomalies-" + JOB_ID), + Tuple.tuple(".ml-anomalies-shared", ".ml-anomalies-.write-" + JOB_ID) + ); @Override protected Settings restClientSettings() { @@ -63,32 +70,27 @@ public void waitForMlTemplates() throws Exception { public void testMlIndicesBecomeHidden() throws Exception { if (isRunningAgainstOldCluster()) { // trigger ML indices creation - createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); - openAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); + createAnomalyDetectorJob(JOB_ID); + openAnomalyDetectorJob(JOB_ID); if (getOldClusterVersion().before(Version.V_7_7_0)) { Map indexSettingsMap = contentAsMap(getMlIndicesSettings()); Map aliasesMap = contentAsMap(getMlAliases()); + assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4)))); for (Map.Entry e : indexSettingsMap.entrySet()) { String indexName = e.getKey(); @SuppressWarnings("unchecked") Map settings = (Map) e.getValue(); assertThat(settings, is(notNullValue())); - assertThat("Index " + indexName + " expected not to be hidden but was", + assertThat("Index " + indexName + " expected not to be hidden but was, settings = " + settings, XContentMapValues.extractValue(settings, "settings", "index", "hidden"), is(nullValue())); } - List> expected = - List.of( - Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.READ_ALIAS_NAME), - Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.WRITE_ALIAS_NAME), - Tuple.tuple(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, AnomalyDetectorsIndex.jobStateIndexWriteAlias()) - ); - for (Tuple indexAndAlias : expected) { + for (Tuple indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) { assertThat( - indexAndAlias + " should not be hidden", + indexAndAlias + " expected not be hidden but was, aliasesMap = " + aliasesMap, XContentMapValues.extractValue(aliasesMap, indexAndAlias.v1(), "aliases", indexAndAlias.v2(), "is_hidden"), is(nullValue())); } @@ -97,29 +99,20 @@ public void testMlIndicesBecomeHidden() throws Exception { Map indexSettingsMap = contentAsMap(getMlIndicesSettings()); Map aliasesMap = contentAsMap(getMlAliases()); + assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4)))); for (Map.Entry e : indexSettingsMap.entrySet()) { String indexName = e.getKey(); - // .ml-config is supposed to be a system index, *not* a hidden index - if (".ml-config".equals(indexName)) { - continue; - } @SuppressWarnings("unchecked") Map settings = (Map) e.getValue(); assertThat(settings, is(notNullValue())); - assertThat("Index " + indexName + " expected to be hidden but wasn't", + assertThat("Index " + indexName + " expected to be hidden but wasn't, settings = " + settings, XContentMapValues.extractValue(settings, "settings", "index", "hidden"), is(equalTo("true"))); } - // TODO: Include all the ML aliases - List> expected = - List.of( - Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.READ_ALIAS_NAME), - Tuple.tuple(AnnotationIndex.INDEX_NAME, AnnotationIndex.WRITE_ALIAS_NAME) - ); - for (Tuple indexAndAlias : expected) { + for (Tuple indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) { assertThat( - indexAndAlias + " should be hidden, " + aliasesMap, + indexAndAlias + " expected to be hidden but wasn't, aliasesMap = " + aliasesMap, XContentMapValues.extractValue(aliasesMap, indexAndAlias.v1(), "aliases", indexAndAlias.v2(), "is_hidden"), is(true)); } @@ -127,7 +120,8 @@ public void testMlIndicesBecomeHidden() throws Exception { } private Response getMlIndicesSettings() throws IOException { - Request getSettingsRequest = new Request("GET", ".ml-*/_settings"); + Request getSettingsRequest = + new Request("GET", ".ml-anomalies-*,.ml-state*,.ml-stats-*,.ml-notifications*,.ml-annotations*/_settings"); getSettingsRequest .setOptions(RequestOptions.DEFAULT.toBuilder() .setWarningsHandler(WarningsHandler.PERMISSIVE) From 428a22b973b2f884d39468c46951243c0cde766a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Thu, 7 Oct 2021 14:03:05 +0200 Subject: [PATCH 17/18] Make MlInitializationServiceIT test wait for the code making indices hidden to execute --- .../ml/integration/MlInitializationServiceIT.java | 7 ++++--- .../xpack/ml/MlInitializationService.java | 13 ++++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java index 9caa1d56e9e8c..a3dc2475ba75a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java @@ -46,7 +46,7 @@ public void setUpMocks() { mlInitializationService = new MlInitializationService(client(), threadPool, mlDailyMaintenanceService, clusterService); } - public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() { + public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() throws Exception { String[] mlHiddenIndexNames = { ".ml-anomalies-7", ".ml-state-000001", @@ -84,6 +84,7 @@ public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() { } mlInitializationService.onMaster(); + assertBusy(() -> assertTrue(mlInitializationService.areMlInternalIndicesHidden())); settingsResponse = client().admin().indices().prepareGetSettings(allIndexNames) @@ -94,14 +95,14 @@ public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() { Settings settings = settingsResponse.getIndexToSettings().get(indexName); assertThat(settings, is(notNullValue())); assertThat( - "Index " + indexName + " expected to be hidden but wasn't", + "Index " + indexName + " expected to be hidden but wasn't, settings = " + settings, settings.getAsBoolean(SETTING_INDEX_HIDDEN, false), is(equalTo(true))); } for (String indexName : otherIndexNames) { Settings settings = settingsResponse.getIndexToSettings().get(indexName); assertThat(settings, is(notNullValue())); assertThat( - "Index " + indexName + " expected not to be hidden but was", + "Index " + indexName + " expected not to be hidden but was, settings = " + settings, settings.getAsBoolean(SETTING_INDEX_HIDDEN, false), is(equalTo(false))); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java index 87b92bd36d9b1..d4f6d69a51738 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java @@ -53,6 +53,7 @@ public class MlInitializationService implements ClusterStateListener { private final Client client; private final ThreadPool threadPool; private final AtomicBoolean isIndexCreationInProgress = new AtomicBoolean(false); + private final AtomicBoolean mlInternalIndicesHidden = new AtomicBoolean(false); private final MlDailyMaintenanceService mlDailyMaintenanceService; @@ -98,7 +99,7 @@ public void beforeStop() { public void onMaster() { mlDailyMaintenanceService.start(); - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(this::makeMlIndicesHidden); + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(this::makeMlInternalIndicesHidden); } public void offMaster() { @@ -140,7 +141,12 @@ MlDailyMaintenanceService getDailyMaintenanceService() { return mlDailyMaintenanceService; } - private void makeMlIndicesHidden() { + /** For testing */ + public boolean areMlInternalIndicesHidden() { + return mlInternalIndicesHidden.get(); + } + + private void makeMlInternalIndicesHidden() { String[] mlHiddenIndexPatterns = MachineLearning.getMlHiddenIndexPatterns(); // Step 5: Handle errors encountered on the way. @@ -150,7 +156,7 @@ private void makeMlIndicesHidden() { logger.error("One or more of the ML internal aliases could not be made hidden."); return; } - logger.debug("All the ML internal indices and aliases were successfully made hidden"); + mlInternalIndicesHidden.set(true); }, e -> logger.error("An error occurred while making ML internal indices and aliases hidden", e) ); @@ -176,6 +182,7 @@ private void makeMlIndicesHidden() { } if (indicesAliasesRequest.getAliasActions().isEmpty()) { logger.debug("There are no ML internal aliases that need to be made hidden, [{}]", getAliasesResponse.getAliases()); + finalListener.onResponse(AcknowledgedResponse.TRUE); return; } String indicesWithNonHiddenAliasesString = From b11436a7307a761c6c2bac60f15a8eb6c4a8e200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Thu, 7 Oct 2021 15:16:16 +0200 Subject: [PATCH 18/18] Take into account that .ml-state-write alias may be attached to .ml-state when the aliases are made hidden --- .../MlHiddenIndicesFullClusterRestartIT.java | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 3bdd3ac7eea99..a4b2528dfa870 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -40,13 +40,13 @@ public class MlHiddenIndicesFullClusterRestartIT extends AbstractFullClusterRestartTestCase { private static final String JOB_ID = "ml-hidden-indices-old-cluster-job"; - private static final List> EXPECTED_INDEX_ALIAS_PAIRS = + private static final List, String>> EXPECTED_INDEX_ALIAS_PAIRS = List.of( - Tuple.tuple(".ml-annotations-6", ".ml-annotations-read"), - Tuple.tuple(".ml-annotations-6", ".ml-annotations-write"), - Tuple.tuple(".ml-state-000001", ".ml-state-write"), - Tuple.tuple(".ml-anomalies-shared", ".ml-anomalies-" + JOB_ID), - Tuple.tuple(".ml-anomalies-shared", ".ml-anomalies-.write-" + JOB_ID) + Tuple.tuple(List.of(".ml-annotations-6"), ".ml-annotations-read"), + Tuple.tuple(List.of(".ml-annotations-6"), ".ml-annotations-write"), + Tuple.tuple(List.of(".ml-state", ".ml-state-000001"), ".ml-state-write"), + Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-" + JOB_ID), + Tuple.tuple(List.of(".ml-anomalies-shared"), ".ml-anomalies-.write-" + JOB_ID) ); @Override @@ -88,11 +88,15 @@ public void testMlIndicesBecomeHidden() throws Exception { is(nullValue())); } - for (Tuple indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) { - assertThat( - indexAndAlias + " expected not be hidden but was, aliasesMap = " + aliasesMap, - XContentMapValues.extractValue(aliasesMap, indexAndAlias.v1(), "aliases", indexAndAlias.v2(), "is_hidden"), - is(nullValue())); + for (Tuple, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) { + List indices = indexAndAlias.v1(); + String alias = indexAndAlias.v2(); + for (String index : indices) { + assertThat( + indexAndAlias + " expected not be hidden but was, aliasesMap = " + aliasesMap, + XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden"), + is(nullValue())); + } } } } else { @@ -110,10 +114,14 @@ public void testMlIndicesBecomeHidden() throws Exception { is(equalTo("true"))); } - for (Tuple indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) { + for (Tuple, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) { + List indices = indexAndAlias.v1(); + String alias = indexAndAlias.v2(); assertThat( indexAndAlias + " expected to be hidden but wasn't, aliasesMap = " + aliasesMap, - XContentMapValues.extractValue(aliasesMap, indexAndAlias.v1(), "aliases", indexAndAlias.v2(), "is_hidden"), + indices.stream() + .anyMatch(index -> + Boolean.TRUE.equals(XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden"))), is(true)); } }