[ML] Preserve alias attributes when making them hidden (#80346)

Author: przemekwitek

x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlInitializationServiceIT.java

@@ -7,24 +7,30 @@
 package org.elasticsearch.xpack.ml.integration;
 
 import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.ml.MachineLearning;
 import org.elasticsearch.xpack.ml.MlDailyMaintenanceService;
 import org.elasticsearch.xpack.ml.MlInitializationService;
 import org.junit.Before;
 
-import java.util.Arrays;
-import java.util.Collections;
+import java.util.List;
 import java.util.stream.Stream;
 
+import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -47,36 +53,33 @@ public void setUpMocks() {
     }
 
     public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() throws Exception {
-        String[] mlHiddenIndexNames = {
+        List<String> mlHiddenIndexNames = List.of(
             ".ml-anomalies-7",
             ".ml-state-000001",
             ".ml-stats-000001",
             ".ml-notifications-000002",
-            ".ml-annotations-6" };
-        String[] otherIndexNames = { "some-index-1", "some-other-index-2" };
-        String[] allIndexNames = Stream.concat(Arrays.stream(mlHiddenIndexNames), Arrays.stream(otherIndexNames)).toArray(String[]::new);
+            ".ml-annotations-000001"
+        );
+        List<String> otherIndexNames = List.of("some-index-1", "some-other-index-2");
+        List<String> allIndexNames = Stream.concat(mlHiddenIndexNames.stream(), otherIndexNames.stream()).collect(toList());
 
         for (String indexName : mlHiddenIndexNames) {
             try {
-                assertAcked(prepareCreate(indexName).setSettings(Collections.singletonMap(SETTING_INDEX_HIDDEN, randomBoolean())));
+                createIndex(indexName, Settings.builder().put(SETTING_INDEX_HIDDEN, randomBoolean()).build());
             } catch (ResourceAlreadyExistsException e) {
-                logger.info("Index " + indexName + "already exists: {}", e.getDetailedMessage());
+                logger.info("Index " + indexName + " already exists: {}", e.getDetailedMessage());
             }
         }
-        createIndex(otherIndexNames);
+        for (String indexName : otherIndexNames) {
+            createIndex(indexName, Settings.EMPTY);
+        }
 
-        GetSettingsResponse settingsResponse = client().admin()
-            .indices()
-            .prepareGetSettings(allIndexNames)
-            .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
-            .get();
-        assertThat(settingsResponse, is(notNullValue()));
+        ImmutableOpenMap<String, Settings> indexToSettings = getIndexToSettingsMap(allIndexNames);
         for (String indexName : mlHiddenIndexNames) {
-            Settings settings = settingsResponse.getIndexToSettings().get(indexName);
-            assertThat(settings, is(notNullValue()));
+            assertThat(indexToSettings.get(indexName), is(notNullValue()));
         }
         for (String indexName : otherIndexNames) {
-            Settings settings = settingsResponse.getIndexToSettings().get(indexName);
+            Settings settings = indexToSettings.get(indexName);
             assertThat(settings, is(notNullValue()));
             assertThat(
                 "Index " + indexName + " expected not to be hidden but was",
@@ -85,17 +88,13 @@ public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() throws Excep
             );
         }
 
+        assertFalse(mlInitializationService.areMlInternalIndicesHidden());
         mlInitializationService.onMaster();
         assertBusy(() -> assertTrue(mlInitializationService.areMlInternalIndicesHidden()));
 
-        settingsResponse = client().admin()
-            .indices()
-            .prepareGetSettings(allIndexNames)
-            .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
-            .get();
-        assertThat(settingsResponse, is(notNullValue()));
+        indexToSettings = getIndexToSettingsMap(allIndexNames);
         for (String indexName : mlHiddenIndexNames) {
-            Settings settings = settingsResponse.getIndexToSettings().get(indexName);
+            Settings settings = indexToSettings.get(indexName);
             assertThat(settings, is(notNullValue()));
             assertThat(
                 "Index " + indexName + " expected to be hidden but wasn't, settings = " + settings,
@@ -104,7 +103,7 @@ public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() throws Excep
             );
         }
         for (String indexName : otherIndexNames) {
-            Settings settings = settingsResponse.getIndexToSettings().get(indexName);
+            Settings settings = indexToSettings.get(indexName);
             assertThat(settings, is(notNullValue()));
             assertThat(
                 "Index " + indexName + " expected not to be hidden but was, settings = " + settings,
@@ -114,6 +113,113 @@ public void testThatMlIndicesBecomeHiddenWhenTheNodeBecomesMaster() throws Excep
         }
     }
 
+    public void testThatMlAliasesBecomeHiddenWhenTheNodeBecomesMaster() throws Exception {
+        List<String> mlHiddenIndexNames = List.of(
+            ".ml-anomalies-7",
+            ".ml-state-000001",
+            ".ml-stats-000001",
+            ".ml-notifications-000002",
+            ".ml-annotations-000001"
+        );
+        for (String indexName : mlHiddenIndexNames) {
+            try {
+                createIndex(indexName, Settings.builder().put(SETTING_INDEX_HIDDEN, randomBoolean()).build());
+            } catch (ResourceAlreadyExistsException e) {
+                logger.info("Index " + indexName + " already exists: {}", e.getDetailedMessage());
+            }
+        }
+
+        IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest().addAliasAction(
+            IndicesAliasesRequest.AliasActions.add().index(".ml-anomalies-7").alias(".ml-anomalies-write").writeIndex(true)
+        )
+            .addAliasAction(
+                IndicesAliasesRequest.AliasActions.add()
+                    .index(".ml-state-000001")
+                    .alias(".ml-state-write")
+                    .filter(QueryBuilders.termQuery("a", "b"))
+            )
+            .addAliasAction(
+                IndicesAliasesRequest.AliasActions.add()
+                    .index(".ml-stats-000001")
+                    .alias(".ml-stats-write")
+                    .indexRouting("some-index-routing")
+            )
+            .addAliasAction(
+                IndicesAliasesRequest.AliasActions.add()
+                    .index(".ml-notifications-000002")
+                    .alias(".ml-notifications-write")
+                    .searchRouting("some-search-routing")
+            )
+            .addAliasAction(
+                IndicesAliasesRequest.AliasActions.add()
+                    .index(".ml-annotations-000001")
+                    .alias(".ml-annotations-write")
+                    .writeIndex(true)
+                    .filter(QueryBuilders.termQuery("a", "b"))
+                    .indexRouting("some-index-routing")
+                    .searchRouting("some-search-routing")
+            );
+        assertAcked(client().admin().indices().aliases(indicesAliasesRequest).get());
+
+        assertFalse(mlInitializationService.areMlInternalIndicesHidden());
+        mlInitializationService.onMaster();
+        assertBusy(() -> assertTrue(mlInitializationService.areMlInternalIndicesHidden()));
+
+        ImmutableOpenMap<String, List<AliasMetadata>> indexToAliasesMap = getIndexToAliasesMap(mlHiddenIndexNames);
+        assertThat("Aliases were: " + indexToAliasesMap, indexToAliasesMap.size(), is(equalTo(5)));
+        assertThat(
+            indexToAliasesMap.get(".ml-anomalies-7"),
+            contains(AliasMetadata.builder(".ml-anomalies-write").isHidden(true).writeIndex(true).build())
+        );
+        assertThat(
+            indexToAliasesMap.get(".ml-state-000001"),
+            contains(AliasMetadata.builder(".ml-state-write").isHidden(true).filter(QueryBuilders.termQuery("a", "b").toString()).build())
+        );
+        assertThat(
+            indexToAliasesMap.get(".ml-stats-000001"),
+            contains(AliasMetadata.builder(".ml-stats-write").isHidden(true).indexRouting("some-index-routing").build())
+        );
+        assertThat(
+            indexToAliasesMap.get(".ml-notifications-000002"),
+            contains(AliasMetadata.builder(".ml-notifications-write").isHidden(true).searchRouting("some-search-routing").build())
+        );
+        assertThat(
+            indexToAliasesMap.get(".ml-annotations-000001"),
+            contains(
+                AliasMetadata.builder(".ml-annotations-read").isHidden(true).build(),
+                AliasMetadata.builder(".ml-annotations-write")
+                    .isHidden(true)
+                    .writeIndex(true)
+                    .filter(QueryBuilders.termQuery("a", "b").toString())
+                    .indexRouting("some-index-routing")
+                    .searchRouting("some-search-routing")
+                    .build()
+            )
+        );
+    }
+
+    private static ImmutableOpenMap<String, Settings> getIndexToSettingsMap(List<String> indexNames) {
+        GetSettingsResponse getSettingsResponse = client().admin()
+            .indices()
+            .prepareGetSettings()
+            .setIndices(indexNames.toArray(String[]::new))
+            .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
+            .get();
+        assertThat(getSettingsResponse, is(notNullValue()));
+        return getSettingsResponse.getIndexToSettings();
+    }
+
+    private static ImmutableOpenMap<String, List<AliasMetadata>> getIndexToAliasesMap(List<String> indexNames) {
+        GetAliasesResponse getAliasesResponse = client().admin()
+            .indices()
+            .prepareGetAliases()
+            .setIndices(indexNames.toArray(String[]::new))
+            .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
+            .get();
+        assertThat(getAliasesResponse, is(notNullValue()));
+        return getAliasesResponse.getAliases();
+    }
+
     @Override
     public Settings indexSettings() {
         return Settings.builder().put(super.indexSettings()).put(IndexMetadata.SETTING_DATA_PATH, (String) null).build();

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlInitializationService.java

@@ -183,19 +183,12 @@ private void makeMlInternalIndicesHidden() {
             IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
             for (ObjectObjectCursor<String, List<AliasMetadata>> entry : getAliasesResponse.getAliases()) {
                 String index = entry.key;
-                String[] nonHiddenAliases = entry.value.stream()
-                    .filter(metadata -> metadata.isHidden() == null || metadata.isHidden() == false)
-                    .map(AliasMetadata::alias)
-                    .toArray(String[]::new);
-                if (nonHiddenAliases.length == 0) {
-                    continue;
+                for (AliasMetadata existingAliasMetadata : entry.value) {
+                    if (existingAliasMetadata.isHidden() != null && existingAliasMetadata.isHidden()) {
+                        continue;
+                    }
+                    indicesAliasesRequest.addAliasAction(aliasReplacementAction(index, existingAliasMetadata));
                 }
-                indicesAliasesRequest.addAliasAction(
-                    IndicesAliasesRequest.AliasActions.add()
-                        .index(index)
-                        .aliases(entry.value.stream().map(AliasMetadata::alias).toArray(String[]::new))
-                        .isHidden(true)
-                );
             }
             if (indicesAliasesRequest.getAliasActions().isEmpty()) {
                 logger.debug("There are no ML internal aliases that need to be made hidden, [{}]", getAliasesResponse.getAliases());
@@ -246,4 +239,25 @@ private void makeMlInternalIndicesHidden() {
             .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN);
         client.admin().indices().getSettings(getSettingsRequest, getSettingsListener);
     }
+
+    private static IndicesAliasesRequest.AliasActions aliasReplacementAction(String index, AliasMetadata existingAliasMetadata) {
+        IndicesAliasesRequest.AliasActions addReplacementAliasAction = IndicesAliasesRequest.AliasActions.add()
+            .index(index)
+            .aliases(existingAliasMetadata.getAlias())
+            .isHidden(true);
+        // Be sure to preserve all attributes apart from is_hidden
+        if (existingAliasMetadata.writeIndex() != null) {
+            addReplacementAliasAction.writeIndex(existingAliasMetadata.writeIndex());
+        }
+        if (existingAliasMetadata.filteringRequired()) {
+            addReplacementAliasAction.filter(existingAliasMetadata.filter().string());
+        }
+        if (existingAliasMetadata.indexRouting() != null) {
+            addReplacementAliasAction.indexRouting(existingAliasMetadata.indexRouting());
+        }
+        if (existingAliasMetadata.searchRouting() != null) {
+            addReplacementAliasAction.searchRouting(existingAliasMetadata.searchRouting());
+        }
+        return addReplacementAliasAction;
+    }
 }