Skip to content

Commit 813609b

Browse files
jakelandisjakelandis
authored
Ensure that .watcher-history-11* template is in installed prior to use (#56734)
WatcherIndexTemplateRegistry as of #52962 requires all nodes to be on 7.7.0 before it allows the version 11 index template to be installed. While in a mixed cluster, nothing prevents Watcher from running on the new host before the all of the nodes are on 7.7.0. This will result in the .watcher-history-11* index without the proper mappings. Without the proper mapping a single document (for a large watch) can exceed the default 1000 field limit and cause error to show in the logs. This commit ensures the same logic for writing to the index is applied as for installing the template. In a mixed cluster, the `10` index template will continue to be written. Only once all of nodes are on 7.7.0+ will the `11` index template be installed and used. closes #56732
1 parent e492c23 commit 813609b

File tree

10 files changed

+63
-34
lines changed

10 files changed

+63
-34
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ private void addTemplatesIfMissing(ClusterState state) {
154154
if (creationCheck.compareAndSet(false, true)) {
155155
IndexTemplateMetadata currentTemplate = state.metadata().getTemplates().get(templateName);
156156
if (Objects.isNull(currentTemplate)) {
157-
logger.debug("adding index template [{}] for [{}], because it doesn't exist", templateName, getOrigin());
157+
logger.info("adding index template [{}] for [{}], because it doesn't exist", templateName, getOrigin());
158158
putTemplate(newTemplate, creationCheck);
159159
} else if (Objects.isNull(currentTemplate.getVersion()) || newTemplate.getVersion() > currentTemplate.getVersion()) {
160160
// IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/history/HistoryStoreField.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
*/
66
package org.elasticsearch.xpack.core.watcher.history;
77

8+
import org.elasticsearch.Version;
9+
import org.elasticsearch.cluster.ClusterState;
810
import org.elasticsearch.common.time.DateFormatter;
911
import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField;
1012

@@ -14,12 +16,18 @@ public final class HistoryStoreField {
1416

1517
public static final String INDEX_PREFIX = ".watcher-history-";
1618
public static final String INDEX_PREFIX_WITH_TEMPLATE = INDEX_PREFIX + WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION + "-";
19+
public static final String INDEX_PREFIX_WITH_TEMPLATE_10 = INDEX_PREFIX +
20+
WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION_10 + "-";
1721
private static final DateFormatter indexTimeFormat = DateFormatter.forPattern("yyyy.MM.dd");
1822

1923
/**
2024
* Calculates the correct history index name for a given time
2125
*/
22-
public static String getHistoryIndexNameForTime(ZonedDateTime time) {
23-
return INDEX_PREFIX_WITH_TEMPLATE + indexTimeFormat.format(time);
26+
public static String getHistoryIndexNameForTime(ZonedDateTime time, ClusterState state) {
27+
if (state == null || state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) {
28+
return INDEX_PREFIX_WITH_TEMPLATE + indexTimeFormat.format(time);
29+
} else {
30+
return INDEX_PREFIX_WITH_TEMPLATE_10 + indexTimeFormat.format(time);
31+
}
2432
}
2533
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ public final class WatcherIndexTemplateRegistryField {
1818
// version 11: watch history indices are hidden
1919
// Note: if you change this, also inform the kibana team around the watcher-ui
2020
public static final int INDEX_TEMPLATE_VERSION = 11;
21+
public static final int INDEX_TEMPLATE_VERSION_10 = 10;
2122
public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
22-
public static final String HISTORY_TEMPLATE_NAME_10 = ".watch-history-10";
23+
public static final String HISTORY_TEMPLATE_NAME_10 = ".watch-history-" + INDEX_TEMPLATE_VERSION_10;
2324
public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
2425
public static final String HISTORY_TEMPLATE_NAME_NO_ILM_10 = ".watch-history-no-ilm-10";
2526
public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1395,7 +1395,7 @@ public void testWatcherAdminRole() {
13951395
assertThat(role.indices().allowedIndicesMatcher(IndexAction.NAME).test("foo"), is(false));
13961396

13971397
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
1398-
String historyIndex = HistoryStoreField.getHistoryIndexNameForTime(now);
1398+
String historyIndex = HistoryStoreField.getHistoryIndexNameForTime(now, null);
13991399
for (String index : new String[]{ Watch.INDEX, historyIndex, TriggeredWatchStoreField.INDEX_NAME }) {
14001400
assertOnlyReadAllowed(role, index);
14011401
}
@@ -1429,7 +1429,7 @@ public void testWatcherUserRole() {
14291429
assertThat(role.indices().allowedIndicesMatcher(IndexAction.NAME).test(TriggeredWatchStoreField.INDEX_NAME), is(false));
14301430

14311431
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
1432-
String historyIndex = HistoryStoreField.getHistoryIndexNameForTime(now);
1432+
String historyIndex = HistoryStoreField.getHistoryIndexNameForTime(now, null);
14331433
for (String index : new String[]{ Watch.INDEX, historyIndex }) {
14341434
assertOnlyReadAllowed(role, index);
14351435
}

‎x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
386386
.setConcurrentRequests(SETTING_BULK_CONCURRENT_REQUESTS.get(settings))
387387
.build();
388388

389-
HistoryStore historyStore = new HistoryStore(bulkProcessor);
389+
HistoryStore historyStore = new HistoryStore(bulkProcessor, clusterService::state);
390390

391391
// schedulers
392392
final Set<Schedule.Parser> scheduleParsers = new HashSet<>();
@@ -623,14 +623,14 @@ static void validAutoCreateIndex(Settings settings, Logger logger) {
623623
indices.add(".watches");
624624
indices.add(".triggered_watches");
625625
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
626-
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now));
627-
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusDays(1)));
628-
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(1)));
629-
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(2)));
630-
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(3)));
631-
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(4)));
632-
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(5)));
633-
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(6)));
626+
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now, null));
627+
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusDays(1), null));
628+
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(1), null));
629+
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(2), null));
630+
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(3), null));
631+
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(4), null));
632+
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(5), null));
633+
indices.add(HistoryStoreField.getHistoryIndexNameForTime(now.plusMonths(6), null));
634634
for (String index : indices) {
635635
boolean matched = false;
636636
for (String match : matches) {

‎x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/execution/ExecutionService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,7 @@ private void executeAsync(WatchExecutionContext ctx, final TriggeredWatch trigge
452452
* Any existing watchRecord will be overwritten.
453453
*/
454454
private void forcePutHistory(WatchRecord watchRecord) {
455-
String index = HistoryStoreField.getHistoryIndexNameForTime(watchRecord.triggerEvent().triggeredTime());
455+
String index = HistoryStoreField.getHistoryIndexNameForTime(watchRecord.triggerEvent().triggeredTime(), clusterService.state());
456456
try {
457457
try (XContentBuilder builder = XContentFactory.jsonBuilder();
458458
ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(WATCHER_ORIGIN)) {

‎x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/history/HistoryStore.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,19 @@ public class HistoryStore {
3131
private static final Logger logger = LogManager.getLogger(HistoryStore.class);
3232

3333
private final BulkProcessor bulkProcessor;
34+
private final Supplier<ClusterState> clusterStateSupplier;
3435

35-
public HistoryStore(BulkProcessor bulkProcessor) {
36+
public HistoryStore(BulkProcessor bulkProcessor, Supplier<ClusterState> clusterStateSupplier) {
3637
this.bulkProcessor = bulkProcessor;
38+
this.clusterStateSupplier = clusterStateSupplier;
3739
}
3840

3941
/**
4042
* Stores the specified watchRecord.
4143
* If the specified watchRecord already was stored this call will fail with a version conflict.
4244
*/
4345
public void put(WatchRecord watchRecord) throws Exception {
44-
String index = HistoryStoreField.getHistoryIndexNameForTime(watchRecord.triggerEvent().triggeredTime());
46+
String index = HistoryStoreField.getHistoryIndexNameForTime(watchRecord.triggerEvent().triggeredTime(), clusterStateSupplier.get());
4547
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
4648
watchRecord.toXContent(builder, WatcherParams.HIDE_SECRETS);
4749

@@ -58,7 +60,7 @@ public void put(WatchRecord watchRecord) throws Exception {
5860
* Any existing watchRecord will be overwritten.
5961
*/
6062
public void forcePut(WatchRecord watchRecord) {
61-
String index = HistoryStoreField.getHistoryIndexNameForTime(watchRecord.triggerEvent().triggeredTime());
63+
String index = HistoryStoreField.getHistoryIndexNameForTime(watchRecord.triggerEvent().triggeredTime(), clusterStateSupplier.get());
6264
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
6365
watchRecord.toXContent(builder, WatcherParams.HIDE_SECRETS);
6466

@@ -78,7 +80,7 @@ public void forcePut(WatchRecord watchRecord) {
7880
* @return true, if history store is ready to be started
7981
*/
8082
public static boolean validate(ClusterState state) {
81-
String currentIndex = HistoryStoreField.getHistoryIndexNameForTime(ZonedDateTime.now(ZoneOffset.UTC));
83+
String currentIndex = HistoryStoreField.getHistoryIndexNameForTime(ZonedDateTime.now(ZoneOffset.UTC), state);
8284
IndexMetadata indexMetadata = WatchStoreUtils.getConcreteIndex(currentIndex, state.metadata());
8385
return indexMetadata == null || (indexMetadata.getState() == IndexMetadata.State.OPEN &&
8486
state.routingTable().index(indexMetadata.getIndex()).allPrimaryShardsActive());

‎x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import org.apache.http.HttpStatus;
99
import org.elasticsearch.ElasticsearchException;
10+
import org.elasticsearch.Version;
1011
import org.elasticsearch.action.ActionListener;
1112
import org.elasticsearch.action.DocWriteRequest.OpType;
1213
import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -16,6 +17,8 @@
1617
import org.elasticsearch.action.index.IndexRequest;
1718
import org.elasticsearch.action.index.IndexResponse;
1819
import org.elasticsearch.client.Client;
20+
import org.elasticsearch.cluster.ClusterState;
21+
import org.elasticsearch.cluster.node.DiscoveryNodes;
1922
import org.elasticsearch.common.settings.MockSecureSettings;
2023
import org.elasticsearch.common.settings.Settings;
2124
import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -43,11 +46,13 @@
4346
import java.time.Instant;
4447
import java.time.ZoneOffset;
4548
import java.time.ZonedDateTime;
49+
import java.util.Arrays;
4650

4751
import static java.util.Collections.emptyMap;
4852
import static java.util.Collections.singletonMap;
4953
import static org.elasticsearch.xpack.core.watcher.history.HistoryStoreField.getHistoryIndexNameForTime;
5054
import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION;
55+
import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION_10;
5156
import static org.hamcrest.Matchers.containsString;
5257
import static org.hamcrest.Matchers.hasSize;
5358
import static org.hamcrest.Matchers.instanceOf;
@@ -63,24 +68,30 @@ public class HistoryStoreTests extends ESTestCase {
6368

6469
private HistoryStore historyStore;
6570
private Client client;
71+
private ClusterState clusterState;
72+
private DiscoveryNodes discoveryNodes;
6673

6774
@Before
6875
public void init() {
6976
Settings settings = Settings.builder().put("node.name", randomAlphaOfLength(10)).build();
7077
client = mock(Client.class);
78+
clusterState = mock(ClusterState.class);
79+
discoveryNodes = mock(DiscoveryNodes.class);
7180
ThreadPool threadPool = mock(ThreadPool.class);
7281
when(client.threadPool()).thenReturn(threadPool);
7382
when(client.settings()).thenReturn(settings);
7483
when(threadPool.getThreadContext()).thenReturn(new ThreadContext(settings));
84+
when(clusterState.nodes()).thenReturn(discoveryNodes);
85+
when(discoveryNodes.getMinNodeVersion()).thenReturn(randomFrom(Arrays.asList(Version.V_7_0_0, Version.V_7_7_0)));
7586
BulkProcessor.Listener listener = mock(BulkProcessor.Listener.class);
7687
BulkProcessor bulkProcessor = BulkProcessor.builder(client::bulk, listener).setConcurrentRequests(0).setBulkActions(1).build();
77-
historyStore = new HistoryStore(bulkProcessor);
88+
historyStore = new HistoryStore(bulkProcessor, () -> clusterState);
7889
}
7990

8091
public void testPut() throws Exception {
8192
ZonedDateTime now = Instant.ofEpochMilli(0).atZone(ZoneOffset.UTC);
8293
Wid wid = new Wid("_name", now);
83-
String index = getHistoryIndexNameForTime(now);
94+
String index = getHistoryIndexNameForTime(now, clusterState);
8495
ScheduleTriggerEvent event = new ScheduleTriggerEvent(wid.watchId(), now, now);
8596
WatchRecord watchRecord = new WatchRecord.MessageWatchRecord(wid, event, ExecutionState.EXECUTED, null, randomAlphaOfLength(10));
8697

@@ -105,15 +116,11 @@ public void testPut() throws Exception {
105116
}
106117

107118
public void testIndexNameGeneration() {
108-
String indexTemplateVersion = Integer.toString(INDEX_TEMPLATE_VERSION);
109-
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli((long) 0).atZone(ZoneOffset.UTC)),
110-
equalTo(".watcher-history-"+ indexTemplateVersion +"-1970.01.01"));
111-
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(100000000000L).atZone(ZoneOffset.UTC)),
112-
equalTo(".watcher-history-" + indexTemplateVersion + "-1973.03.03"));
113-
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(1416582852000L).atZone(ZoneOffset.UTC)),
114-
equalTo(".watcher-history-" + indexTemplateVersion + "-2014.11.21"));
115-
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(2833165811000L).atZone(ZoneOffset.UTC)),
116-
equalTo(".watcher-history-" + indexTemplateVersion + "-2059.10.12"));
119+
when(discoveryNodes.getMinNodeVersion()).thenReturn(Version.V_7_7_0);
120+
assertHistoryIndexName(Integer.toString(INDEX_TEMPLATE_VERSION));
121+
122+
when(discoveryNodes.getMinNodeVersion()).thenReturn(Version.V_7_0_0);
123+
assertHistoryIndexName(Integer.toString(INDEX_TEMPLATE_VERSION_10));
117124
}
118125

119126
public void testStoreWithHideSecrets() throws Exception {
@@ -179,4 +186,15 @@ public void testStoreWithHideSecrets() throws Exception {
179186
assertThat(indexedJson, containsString(username));
180187
assertThat(indexedJson, not(containsString(password)));
181188
}
189+
190+
private void assertHistoryIndexName(String indexTemplateVersion){
191+
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli((long) 0).atZone(ZoneOffset.UTC), clusterState),
192+
equalTo(".watcher-history-" + indexTemplateVersion + "-1970.01.01"));
193+
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(100000000000L).atZone(ZoneOffset.UTC), clusterState),
194+
equalTo(".watcher-history-" + indexTemplateVersion + "-1973.03.03"));
195+
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(1416582852000L).atZone(ZoneOffset.UTC), clusterState),
196+
equalTo(".watcher-history-" + indexTemplateVersion + "-2014.11.21"));
197+
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(2833165811000L).atZone(ZoneOffset.UTC), clusterState),
198+
equalTo(".watcher-history-" + indexTemplateVersion + "-2059.10.12"));
199+
}
182200
}

‎x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ private void createWatcherIndicesOrAliases() throws Exception {
276276
assertAcked(client().admin().indices().prepareCreate(triggeredWatchIndexName));
277277
}
278278

279-
String historyIndex = HistoryStoreField.getHistoryIndexNameForTime(ZonedDateTime.now(ZoneOffset.UTC));
279+
String historyIndex = HistoryStoreField.getHistoryIndexNameForTime(ZonedDateTime.now(ZoneOffset.UTC), null);
280280
assertAcked(client().admin().indices().prepareCreate(historyIndex));
281281
logger.info("creating watch history index [{}]", historyIndex);
282282
ensureGreen(historyIndex, watchIndexName, triggeredWatchIndexName);

‎x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public void testLoadMalformedWatchRecord() throws Exception {
7878
Wid wid = new Wid("_id", now);
7979
ScheduleTriggerEvent event = new ScheduleTriggerEvent("_id", now, now);
8080
ExecutableCondition condition = InternalAlwaysCondition.INSTANCE;
81-
String index = HistoryStoreField.getHistoryIndexNameForTime(now);
81+
String index = HistoryStoreField.getHistoryIndexNameForTime(now, null);
8282
client().prepareIndex().setIndex(index).setId(wid.value())
8383
.setSource(jsonBuilder().startObject()
8484
.startObject(WatchRecord.TRIGGER_EVENT.getPreferredName())
@@ -309,7 +309,7 @@ public void testWatchRecordSavedTwice() throws Exception {
309309
}
310310
LocalDateTime localDateTime = LocalDateTime.of(2015, 11, 5, 0, 0, 0, 0);
311311
ZonedDateTime triggeredTime = ZonedDateTime.of(localDateTime,ZoneOffset.UTC);
312-
final String watchRecordIndex = HistoryStoreField.getHistoryIndexNameForTime(triggeredTime);
312+
final String watchRecordIndex = HistoryStoreField.getHistoryIndexNameForTime(triggeredTime, null);
313313

314314
logger.info("Stopping watcher");
315315
stopWatcher();

0 commit comments

Comments
 (0)