Skip to content

Commit 5402f4c

Browse files
author
David Roberts
committed
Revert "[ML] Use a new annotations index for future annotations (#79151)"
This reverts commit 7d59540.
1 parent 7d59540 commit 5402f4c

File tree

8 files changed

+31
-104
lines changed

8 files changed

+31
-104
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java‎

Lines changed: 12 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
1616
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
1717
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
18-
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
1918
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
2019
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
2120
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -31,7 +30,6 @@
3130
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
3231
import org.elasticsearch.xpack.core.template.TemplateUtils;
3332

34-
import java.util.List;
3533
import java.util.SortedMap;
3634

3735
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
@@ -43,15 +41,8 @@ public class AnnotationIndex {
4341

4442
public static final String READ_ALIAS_NAME = ".ml-annotations-read";
4543
public static final String WRITE_ALIAS_NAME = ".ml-annotations-write";
46-
47-
// Exposed for testing, but always use the aliases in non-test code.
48-
public static final String LATEST_INDEX_NAME = ".ml-annotations-000001";
49-
// Due to historical bugs this index may not have the correct mappings
50-
// in some production clusters. Therefore new annotations should be
51-
// written to the latest index. If we ever switch to another new annotations
52-
// index then this list should be adjusted to include the previous latest
53-
// index.
54-
public static final List<String> OLD_INDEX_NAMES = List.of(".ml-annotations-6");
44+
// Exposed for testing, but always use the aliases in non-test code
45+
public static final String INDEX_NAME = ".ml-annotations-6";
5546

5647
private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
5748

@@ -94,18 +85,12 @@ public static void createAnnotationsIndexIfNecessary(Client client, ClusterState
9485
finalListener::onFailure);
9586

9687
final ActionListener<Boolean> createAliasListener = ActionListener.wrap(success -> {
97-
final IndicesAliasesRequestBuilder requestBuilder =
88+
final IndicesAliasesRequest request =
9889
client.admin().indices().prepareAliases()
99-
.addAliasAction(IndicesAliasesRequest.AliasActions.add()
100-
.index(LATEST_INDEX_NAME).alias(READ_ALIAS_NAME).isHidden(true))
101-
.addAliasAction(IndicesAliasesRequest.AliasActions.add()
102-
.index(LATEST_INDEX_NAME).alias(WRITE_ALIAS_NAME).isHidden(true));
103-
for (String oldIndexName : OLD_INDEX_NAMES) {
104-
if (state.getMetadata().getIndicesLookup().containsKey(oldIndexName)) {
105-
requestBuilder.removeAlias(oldIndexName, WRITE_ALIAS_NAME);
106-
}
107-
}
108-
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, requestBuilder.request(),
90+
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME).isHidden(true))
91+
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(WRITE_ALIAS_NAME).isHidden(true))
92+
.request();
93+
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
10994
ActionListener.<AcknowledgedResponse>wrap(
11095
r -> checkMappingsListener.onResponse(r.isAcknowledged()), finalListener::onFailure),
11196
client.admin().indices()::aliases);
@@ -119,18 +104,18 @@ public static void createAnnotationsIndexIfNecessary(Client client, ClusterState
119104
mlLookup.isEmpty() == false && mlLookup.firstKey().startsWith(".ml")) {
120105

121106
// Create the annotations index if it doesn't exist already.
122-
if (mlLookup.containsKey(LATEST_INDEX_NAME) == false) {
107+
if (mlLookup.containsKey(INDEX_NAME) == false) {
123108
logger.debug(
124109
() -> new ParameterizedMessage(
125110
"Creating [{}] because [{}] exists; trace {}",
126-
LATEST_INDEX_NAME,
111+
INDEX_NAME,
127112
mlLookup.firstKey(),
128113
org.elasticsearch.ExceptionsHelper.formatStackTrace(Thread.currentThread().getStackTrace())
129114
)
130115
);
131116

132117
CreateIndexRequest createIndexRequest =
133-
new CreateIndexRequest(LATEST_INDEX_NAME)
118+
new CreateIndexRequest(INDEX_NAME)
134119
.mapping(annotationsMapping())
135120
.settings(Settings.builder()
136121
.put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
@@ -155,14 +140,7 @@ public static void createAnnotationsIndexIfNecessary(Client client, ClusterState
155140
}
156141

157142
// Recreate the aliases if they've gone even though the index still exists.
158-
IndexAbstraction writeAliasDefinition = mlLookup.get(WRITE_ALIAS_NAME);
159-
if (mlLookup.containsKey(READ_ALIAS_NAME) == false || writeAliasDefinition == null) {
160-
createAliasListener.onResponse(true);
161-
return;
162-
}
163-
164-
List<IndexMetadata> writeAliasMetadata = writeAliasDefinition.getIndices();
165-
if (writeAliasMetadata.size() != 1 || LATEST_INDEX_NAME.equals(writeAliasMetadata.get(0).getIndex().getName()) == false) {
143+
if (mlLookup.containsKey(READ_ALIAS_NAME) == false || mlLookup.containsKey(WRITE_ALIAS_NAME) == false) {
166144
createAliasListener.onResponse(true);
167145
return;
168146
}
@@ -176,7 +154,7 @@ public static void createAnnotationsIndexIfNecessary(Client client, ClusterState
176154
finalListener.onResponse(false);
177155
}
178156

179-
public static String annotationsMapping() {
157+
private static String annotationsMapping() {
180158
return TemplateUtils.loadTemplate(
181159
"/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE);
182160
}

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
"_meta" : {
44
"version" : "${xpack.ml.version}"
55
},
6-
"dynamic" : false,
76
"properties" : {
87
"annotation" : {
98
"type" : "text"

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1478,7 +1478,7 @@ public void testMachineLearningAdminRole() {
14781478
assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
14791479
assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT);
14801480
assertOnlyReadAllowed(role, NotificationsIndex.NOTIFICATIONS_INDEX);
1481-
assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.LATEST_INDEX_NAME);
1481+
assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.INDEX_NAME);
14821482

14831483
assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES);
14841484
assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2));
@@ -1632,7 +1632,7 @@ public void testMachineLearningUserRole() {
16321632
assertNoAccessAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
16331633
assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT);
16341634
assertOnlyReadAllowed(role, NotificationsIndex.NOTIFICATIONS_INDEX);
1635-
assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.LATEST_INDEX_NAME);
1635+
assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.INDEX_NAME);
16361636

16371637
assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES);
16381638
assertNoAccessAllowed(role, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2));

‎x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ protected void waitForecastStatus(int maxWaitTimeSeconds,
264264

265265
protected void assertThatNumberOfAnnotationsIsEqualTo(int expectedNumberOfAnnotations) throws IOException {
266266
// Refresh the annotations index so that recently indexed annotation docs are visible.
267-
client().admin().indices().prepareRefresh(AnnotationIndex.LATEST_INDEX_NAME)
267+
client().admin().indices().prepareRefresh(AnnotationIndex.INDEX_NAME)
268268
.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
269269
.execute()
270270
.actionGet();

‎x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AnnotationIndexIT.java‎

Lines changed: 7 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,13 @@
77
package org.elasticsearch.xpack.ml.integration;
88

99
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
10-
11-
import org.elasticsearch.action.admin.indices.alias.Alias;
12-
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
13-
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
1410
import org.elasticsearch.action.index.IndexRequest;
1511
import org.elasticsearch.action.index.IndexResponse;
1612
import org.elasticsearch.action.search.SearchPhaseExecutionException;
1713
import org.elasticsearch.action.search.SearchRequest;
1814
import org.elasticsearch.action.search.SearchResponse;
1915
import org.elasticsearch.action.support.IndicesOptions;
2016
import org.elasticsearch.cluster.metadata.AliasMetadata;
21-
import org.elasticsearch.cluster.metadata.IndexMetadata;
2217
import org.elasticsearch.cluster.service.ClusterService;
2318
import org.elasticsearch.common.collect.ImmutableOpenMap;
2419
import org.elasticsearch.common.settings.Settings;
@@ -31,12 +26,11 @@
3126
import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex;
3227
import org.elasticsearch.xpack.ml.MlSingleNodeTestCase;
3328
import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor;
29+
import org.junit.Before;
3430

35-
import java.util.ArrayList;
3631
import java.util.Collections;
3732
import java.util.List;
3833

39-
import static org.hamcrest.Matchers.containsInAnyOrder;
4034
import static org.hamcrest.Matchers.is;
4135

4236
public class AnnotationIndexIT extends MlSingleNodeTestCase {
@@ -50,6 +44,11 @@ protected Settings nodeSettings() {
5044
return newSettings.build();
5145
}
5246

47+
@Before
48+
public void createComponents() throws Exception {
49+
waitForMlTemplates();
50+
}
51+
5352
public void testNotCreatedWhenNoOtherMlIndices() {
5453

5554
// Ask a few times to increase the chance of failure if the .ml-annotations index is created when no other ML index exists
@@ -72,52 +71,6 @@ public void testCreatedWhenAfterOtherMlIndex() throws Exception {
7271
});
7372
}
7473

75-
public void testAliasesMovedFromOldToNew() throws Exception {
76-
77-
// Create an old annotations index with both read and write aliases pointing at it.
78-
String oldIndex = randomFrom(AnnotationIndex.OLD_INDEX_NAMES);
79-
CreateIndexRequest createIndexRequest =
80-
new CreateIndexRequest(oldIndex)
81-
.mapping(AnnotationIndex.annotationsMapping())
82-
.settings(Settings.builder()
83-
.put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
84-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "1")
85-
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true))
86-
.alias(new Alias(AnnotationIndex.READ_ALIAS_NAME).isHidden(true))
87-
.alias(new Alias(AnnotationIndex.WRITE_ALIAS_NAME).isHidden(true));
88-
client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
89-
90-
// Because the old annotations index name began with .ml, it will trigger the new annotations index to be created.
91-
// When this happens the read alias should be changed to cover both indices, and the write alias should be
92-
// switched to only point at the new index.
93-
assertBusy(() -> {
94-
assertTrue(annotationsIndexExists());
95-
ImmutableOpenMap<String, List<AliasMetadata>> aliases = client().admin().indices()
96-
.prepareGetAliases(AnnotationIndex.READ_ALIAS_NAME, AnnotationIndex.WRITE_ALIAS_NAME)
97-
.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
98-
.get()
99-
.getAliases();
100-
assertNotNull(aliases);
101-
List<String> indicesWithReadAlias = new ArrayList<>();
102-
for (ObjectObjectCursor<String, List<AliasMetadata>> entry : aliases) {
103-
for (AliasMetadata aliasMetadata : entry.value) {
104-
switch (aliasMetadata.getAlias()) {
105-
case AnnotationIndex.WRITE_ALIAS_NAME:
106-
assertThat(entry.key, is(AnnotationIndex.LATEST_INDEX_NAME));
107-
break;
108-
case AnnotationIndex.READ_ALIAS_NAME:
109-
indicesWithReadAlias.add(entry.key);
110-
break;
111-
default:
112-
fail("Found unexpected alias " + aliasMetadata.getAlias() + " on index " + entry.key);
113-
break;
114-
}
115-
}
116-
}
117-
assertThat(indicesWithReadAlias, containsInAnyOrder(oldIndex, AnnotationIndex.LATEST_INDEX_NAME));
118-
});
119-
}
120-
12174
public void testNotCreatedWhenAfterOtherMlIndexAndUpgradeInProgress() throws Exception {
12275

12376
client().execute(SetUpgradeModeAction.INSTANCE, new SetUpgradeModeAction.Request(true)).actionGet();
@@ -170,7 +123,7 @@ public void testNotCreatedWhenAfterOtherMlIndexAndResetInProgress() throws Excep
170123
}
171124

172125
private boolean annotationsIndexExists() {
173-
return ESIntegTestCase.indexExists(AnnotationIndex.LATEST_INDEX_NAME, client());
126+
return ESIntegTestCase.indexExists(AnnotationIndex.INDEX_NAME, client());
174127
}
175128

176129
private int numberOfAnnotationsAliases() {

‎x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -731,7 +731,7 @@ private QueryPage<ModelSnapshot> getModelSnapshots() throws Exception {
731731

732732
private List<Annotation> getAnnotations() throws Exception {
733733
// Refresh the annotations index so that recently indexed annotation docs are visible.
734-
client().admin().indices().prepareRefresh(AnnotationIndex.LATEST_INDEX_NAME)
734+
client().admin().indices().prepareRefresh(AnnotationIndex.INDEX_NAME)
735735
.setIndicesOptions(IndicesOptions.STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED)
736736
.execute()
737737
.actionGet();

‎x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,6 @@
9292
import java.util.function.Consumer;
9393

9494
import static org.elasticsearch.action.support.master.MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT;
95-
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN;
9695
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
9796
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
9897
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
@@ -185,23 +184,21 @@ public void setup() throws Exception {
185184
Settings.builder()
186185
.put(SETTING_NUMBER_OF_SHARDS, 1)
187186
.put(SETTING_NUMBER_OF_REPLICAS, 0)
188-
.put(SETTING_INDEX_HIDDEN, true)
189187
.put(SETTING_VERSION_CREATED, Version.CURRENT)
190188
.build())
191-
.putAlias(AliasMetadata.builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).isHidden(true).build())
189+
.putAlias(AliasMetadata.builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).build())
192190
.build())
193191
.fPut(
194-
AnnotationIndex.LATEST_INDEX_NAME,
195-
IndexMetadata.builder(AnnotationIndex.LATEST_INDEX_NAME)
192+
AnnotationIndex.INDEX_NAME,
193+
IndexMetadata.builder(AnnotationIndex.INDEX_NAME)
196194
.settings(
197195
Settings.builder()
198196
.put(SETTING_NUMBER_OF_SHARDS, 1)
199197
.put(SETTING_NUMBER_OF_REPLICAS, 0)
200-
.put(SETTING_INDEX_HIDDEN, true)
201198
.put(SETTING_VERSION_CREATED, Version.CURRENT)
202199
.build())
203-
.putAlias(AliasMetadata.builder(AnnotationIndex.READ_ALIAS_NAME).isHidden(true).build())
204-
.putAlias(AliasMetadata.builder(AnnotationIndex.WRITE_ALIAS_NAME).isHidden(true).build())
200+
.putAlias(AliasMetadata.builder(AnnotationIndex.READ_ALIAS_NAME).build())
201+
.putAlias(AliasMetadata.builder(AnnotationIndex.WRITE_ALIAS_NAME).build())
205202
.build())
206203
.build())
207204
.build();

‎x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,6 @@ setup:
157157
indices.get_mapping:
158158
index: .ml-annotations-write
159159

160-
- match: { \.ml-annotations-000001.mappings.properties.type.type: "keyword" }
161-
- match: { \.ml-annotations-000001.mappings.properties.event.type: "keyword" }
162-
- match: { \.ml-annotations-000001.mappings.properties.detector_index.type: "integer" }
160+
- match: { \.ml-annotations-6.mappings.properties.type.type: "keyword" }
161+
- match: { \.ml-annotations-6.mappings.properties.event.type: "keyword" }
162+
- match: { \.ml-annotations-6.mappings.properties.detector_index.type: "integer" }

0 commit comments

Comments
 (0)