Skip to content

Commit 879700a

Browse files
williamrandolphwilliamrandolph
authored
Add system data streams to feature state snapshots (#75902) (#76568)
* Add system data streams to feature state snapshots (#75902) Add system data streams to the "snapshot feature state" code block, so that if we're snapshotting a feature by name we grab that feature's system data streams too. Handle these data streams on the restore side as well. * Add system data streams to feature state snapshots * Don't pass system data streams through index name resolution * Don't add no-op features to snapshots * Hook in system data streams for snapshot restoration
1 parent 5ddb217 commit 879700a

File tree

4 files changed

+186
-41
lines changed

4 files changed

+186
-41
lines changed

‎server/src/main/java/org/elasticsearch/indices/SystemDataStreamDescriptor.java‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,21 @@
88

99
package org.elasticsearch.indices;
1010

11+
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
1112
import org.elasticsearch.cluster.metadata.ComponentTemplate;
1213
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
1314
import org.elasticsearch.cluster.metadata.DataStream;
15+
import org.elasticsearch.cluster.metadata.Metadata;
1416

17+
import java.util.ArrayList;
1518
import java.util.Collections;
1619
import java.util.HashMap;
1720
import java.util.List;
1821
import java.util.Map;
1922
import java.util.Objects;
2023

24+
import static org.elasticsearch.indices.AssociatedIndexDescriptor.buildAutomaton;
25+
2126
/**
2227
* Describes a {@link DataStream} that is reserved for use by a system component. The data stream will be managed by the system and also
2328
* protected by the system against user modification so that system features are not broken by inadvertent user operations.
@@ -31,6 +36,7 @@ public class SystemDataStreamDescriptor {
3136
private final Map<String, ComponentTemplate> componentTemplates;
3237
private final List<String> allowedElasticProductOrigins;
3338
private final ExecutorNames executorNames;
39+
private final CharacterRunAutomaton characterRunAutomaton;
3440

3541
/**
3642
* Creates a new descriptor for a system data descriptor
@@ -73,12 +79,31 @@ public SystemDataStreamDescriptor(String dataStreamName, String description, Typ
7379
this.executorNames = Objects.nonNull(executorNames)
7480
? executorNames
7581
: ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS;
82+
83+
this.characterRunAutomaton = new CharacterRunAutomaton(
84+
buildAutomaton(backingIndexPatternForDataStream(this.dataStreamName)));
7685
}
7786

7887
public String getDataStreamName() {
7988
return dataStreamName;
8089
}
8190

91+
/**
92+
* Retrieve backing indices for this system data stream
93+
* @param metadata Metadata in which to look for indices
94+
* @return List of names of backing indices
95+
*/
96+
public List<String> getBackingIndexNames(Metadata metadata) {
97+
ArrayList<String> matchingIndices = new ArrayList<>();
98+
metadata.indices().keysIt().forEachRemaining(indexName -> {
99+
if (this.characterRunAutomaton.run(indexName)) {
100+
matchingIndices.add(indexName);
101+
}
102+
});
103+
104+
return Collections.unmodifiableList(matchingIndices);
105+
}
106+
82107
public String getDescription() {
83108
return description;
84109
}
@@ -92,7 +117,11 @@ public boolean isExternal() {
92117
}
93118

94119
public String getBackingIndexPattern() {
95-
return DataStream.BACKING_INDEX_PREFIX + getDataStreamName() + "-*";
120+
return backingIndexPatternForDataStream(getDataStreamName());
121+
}
122+
123+
private static String backingIndexPatternForDataStream(String dataStream) {
124+
return DataStream.BACKING_INDEX_PREFIX + dataStream + "-*";
96125
}
97126

98127
public List<String> getAllowedElasticProductOrigins() {

‎server/src/main/java/org/elasticsearch/snapshots/RestoreService.java‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
import org.elasticsearch.index.shard.IndexShard;
6969
import org.elasticsearch.index.shard.ShardId;
7070
import org.elasticsearch.indices.ShardLimitValidator;
71+
import org.elasticsearch.indices.SystemDataStreamDescriptor;
7172
import org.elasticsearch.indices.SystemIndices;
7273
import org.elasticsearch.repositories.IndexId;
7374
import org.elasticsearch.repositories.RepositoriesService;
@@ -317,13 +318,40 @@ private void startRestore(
317318
Collections.addAll(requestIndices, indicesInRequest);
318319
}
319320

321+
// Determine system indices to restore from requested feature states
322+
final Map<String, List<String>> featureStatesToRestore = getFeatureStatesToRestore(request, snapshotInfo, snapshot);
323+
final Set<String> featureStateIndices = featureStatesToRestore.values()
324+
.stream()
325+
.flatMap(Collection::stream)
326+
.collect(Collectors.toSet());
327+
328+
final Map<String, SystemIndices.Feature> featureSet = systemIndices.getFeatures();
329+
final Set<String> featureStateDataStreams = featureStatesToRestore.keySet().stream().filter(featureName -> {
330+
if (featureSet.containsKey(featureName)) {
331+
return true;
332+
}
333+
logger.warn(
334+
() -> new ParameterizedMessage(
335+
"Restoring snapshot[{}] skipping feature [{}] because it is not available in this cluster",
336+
snapshotInfo.snapshotId(),
337+
featureName
338+
)
339+
);
340+
return false;
341+
})
342+
.map(name -> systemIndices.getFeatures().get(name))
343+
.flatMap(feature -> feature.getDataStreamDescriptors().stream())
344+
.map(SystemDataStreamDescriptor::getDataStreamName)
345+
.collect(Collectors.toSet());
346+
320347
// Get data stream metadata for requested data streams
321348
Tuple<Map<String, DataStream>, Map<String, DataStreamAlias>> result = getDataStreamsToRestore(
322349
repository,
323350
snapshotId,
324351
snapshotInfo,
325352
globalMetadata,
326-
requestIndices,
353+
// include system data stream names in argument to this method
354+
Stream.concat(requestIndices.stream(), featureStateDataStreams.stream()).collect(Collectors.toList()),
327355
request.includeAliases()
328356
);
329357
Map<String, DataStream> dataStreamsToRestore = result.v1();
@@ -340,13 +368,6 @@ private void startRestore(
340368
.collect(Collectors.toSet());
341369
requestIndices.addAll(dataStreamIndices);
342370

343-
// Determine system indices to restore from requested feature states
344-
final Map<String, List<String>> featureStatesToRestore = getFeatureStatesToRestore(request, snapshotInfo, snapshot);
345-
final Set<String> featureStateIndices = featureStatesToRestore.values()
346-
.stream()
347-
.flatMap(Collection::stream)
348-
.collect(Collectors.toSet());
349-
350371
// Resolve the indices that were directly requested
351372
final List<String> requestedIndicesInSnapshot = filterIndices(
352373
snapshotInfo.indices(),

‎server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java‎

Lines changed: 37 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@
7171
import org.elasticsearch.core.Tuple;
7272
import org.elasticsearch.index.Index;
7373
import org.elasticsearch.index.shard.ShardId;
74-
import org.elasticsearch.indices.AssociatedIndexDescriptor;
74+
import org.elasticsearch.indices.SystemDataStreamDescriptor;
7575
import org.elasticsearch.indices.SystemIndices;
7676
import org.elasticsearch.repositories.IndexId;
7777
import org.elasticsearch.repositories.RepositoriesService;
@@ -495,36 +495,45 @@ public ClusterState execute(ClusterState currentState) {
495495
// Store newSnapshot here to be processed in clusterStateProcessed
496496
List<String> indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request));
497497

498-
final List<SnapshotFeatureInfo> featureStates;
498+
final Set<SnapshotFeatureInfo> featureStates = new HashSet<>();
499+
final Set<String> systemDataStreamNames = new HashSet<>();
499500
// if we have any feature states in the snapshot, we add their required indices to the snapshot indices if they haven't
500501
// been requested by the request directly
501-
if (featureStatesSet.isEmpty()) {
502-
featureStates = Collections.emptyList();
503-
} else {
504-
final Set<String> indexNames = new HashSet<>(indices);
505-
featureStates = featureStatesSet.stream()
506-
.map(
507-
feature -> new SnapshotFeatureInfo(
508-
feature,
509-
systemIndexDescriptorMap.get(feature)
510-
.getIndexDescriptors()
511-
.stream()
512-
.flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata()).stream())
513-
.collect(Collectors.toList())
514-
)
515-
)
516-
.filter(featureInfo -> featureInfo.getIndices().isEmpty() == false) // Omit any empty featureStates
517-
.collect(Collectors.toList());
518-
for (SnapshotFeatureInfo featureState : featureStates) {
519-
indexNames.addAll(featureState.getIndices());
520-
}
502+
final Set<String> indexNames = new HashSet<>(indices);
503+
for (String featureName : featureStatesSet) {
504+
SystemIndices.Feature feature = systemIndexDescriptorMap.get(featureName);
521505

522-
// Add all resolved indices from the feature states to the list of indices
523-
for (String feature : featureStatesSet) {
524-
for (AssociatedIndexDescriptor aid : systemIndexDescriptorMap.get(feature).getAssociatedIndexDescriptors()) {
525-
indexNames.addAll(aid.getMatchingIndices(currentState.metadata()));
506+
Set<String> featureSystemIndices = feature.getIndexDescriptors()
507+
.stream()
508+
.flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata()).stream())
509+
.collect(Collectors.toSet());
510+
Set<String> featureAssociatedIndices = feature.getAssociatedIndexDescriptors()
511+
.stream()
512+
.flatMap(descriptor -> descriptor.getMatchingIndices(currentState.metadata()).stream())
513+
.collect(Collectors.toSet());
514+
515+
Set<String> featureSystemDataStreams = new HashSet<>();
516+
Set<String> featureDataStreamBackingIndices = new HashSet<>();
517+
for (SystemDataStreamDescriptor sdd : feature.getDataStreamDescriptors()) {
518+
List<String> backingIndexNames = sdd.getBackingIndexNames(currentState.metadata());
519+
if (backingIndexNames.size() > 0) {
520+
featureDataStreamBackingIndices.addAll(backingIndexNames);
521+
featureSystemDataStreams.add(sdd.getDataStreamName());
526522
}
527523
}
524+
525+
if (featureSystemIndices.size() > 0
526+
|| featureAssociatedIndices.size() > 0
527+
|| featureDataStreamBackingIndices.size() > 0) {
528+
529+
featureStates.add(
530+
new SnapshotFeatureInfo(featureName, Collections.unmodifiableList(new ArrayList<>(featureSystemIndices)))
531+
);
532+
indexNames.addAll(featureSystemIndices);
533+
indexNames.addAll(featureAssociatedIndices);
534+
indexNames.addAll(featureDataStreamBackingIndices);
535+
systemDataStreamNames.addAll(featureSystemDataStreams);
536+
}
528537
indices = Collections.unmodifiableList(new ArrayList<>(indexNames));
529538
}
530539

@@ -533,6 +542,7 @@ public ClusterState execute(ClusterState currentState) {
533542
request.indicesOptions(),
534543
request.indices()
535544
);
545+
dataStreams.addAll(systemDataStreamNames);
536546

537547
logger.trace("[{}][{}] creating snapshot for indices [{}]", repositoryName, snapshotName, indices);
538548

@@ -575,7 +585,7 @@ public ClusterState execute(ClusterState currentState) {
575585
shards,
576586
userMeta,
577587
version,
578-
featureStates
588+
Collections.unmodifiableList(new ArrayList<>(featureStates))
579589
);
580590
return ClusterState.builder(currentState)
581591
.putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(CollectionUtils.appendToCopy(runningSnapshots, newEntry)))

‎x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamSnapshotIT.java‎

Lines changed: 90 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
package org.elasticsearch.datastreams;
99

1010
import org.elasticsearch.action.DocWriteRequest;
11-
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
1211
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
1312
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
1413
import org.elasticsearch.action.index.IndexResponse;
@@ -20,6 +19,7 @@
2019
import org.elasticsearch.plugins.Plugin;
2120
import org.elasticsearch.plugins.SystemIndexPlugin;
2221
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
22+
import org.elasticsearch.snapshots.SnapshotInfo;
2323
import org.elasticsearch.snapshots.mockstore.MockRepository;
2424
import org.elasticsearch.test.ESIntegTestCase;
2525
import org.elasticsearch.threadpool.ThreadPool;
@@ -34,8 +34,11 @@
3434
import java.util.Collections;
3535

3636
import static org.elasticsearch.datastreams.SystemDataStreamSnapshotIT.SystemDataStreamTestPlugin.SYSTEM_DATA_STREAM_NAME;
37+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
3738
import static org.hamcrest.Matchers.arrayWithSize;
39+
import static org.hamcrest.Matchers.empty;
3840
import static org.hamcrest.Matchers.hasSize;
41+
import static org.hamcrest.Matchers.not;
3942
import static org.hamcrest.Matchers.oneOf;
4043

4144
@ESIntegTestCase.ClusterScope(transportClientRatio = 0)
@@ -82,12 +85,91 @@ public void testSystemDataStreamSnapshotIT() throws Exception {
8285
assertTrue(response.getDataStreams().get(0).getDataStream().isSystem());
8386
}
8487

85-
CreateSnapshotResponse createSnapshotResponse = client().admin()
88+
assertSuccessful(
89+
client().admin()
90+
.cluster()
91+
.prepareCreateSnapshot(REPO, SNAPSHOT)
92+
.setWaitForCompletion(true)
93+
.setIncludeGlobalState(false)
94+
.execute()
95+
);
96+
97+
// We have to delete the data stream directly, as the feature reset API doesn't clean up system data streams yet
98+
// See https://github.com/elastic/elasticsearch/issues/75818
99+
{
100+
DeleteDataStreamAction.Request request = new DeleteDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
101+
AcknowledgedResponse response = client().execute(DeleteDataStreamAction.INSTANCE, request).get();
102+
assertTrue(response.isAcknowledged());
103+
}
104+
105+
{
106+
GetIndexResponse indicesRemaining = client().admin().indices().prepareGetIndex().addIndices("_all").get();
107+
assertThat(indicesRemaining.indices(), arrayWithSize(0));
108+
}
109+
110+
RestoreSnapshotResponse restoreSnapshotResponse = client().admin()
86111
.cluster()
87-
.prepareCreateSnapshot(REPO, SNAPSHOT)
112+
.prepareRestoreSnapshot(REPO, SNAPSHOT)
88113
.setWaitForCompletion(true)
89-
.setIncludeGlobalState(false)
114+
.setRestoreGlobalState(false)
90115
.get();
116+
assertEquals(restoreSnapshotResponse.getRestoreInfo().totalShards(), restoreSnapshotResponse.getRestoreInfo().successfulShards());
117+
118+
{
119+
GetDataStreamAction.Request request = new GetDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
120+
GetDataStreamAction.Response response = client().execute(GetDataStreamAction.INSTANCE, request).get();
121+
assertThat(response.getDataStreams(), hasSize(1));
122+
assertTrue(response.getDataStreams().get(0).getDataStream().isSystem());
123+
}
124+
}
125+
126+
public void testSystemDataStreamInFeatureState() throws Exception {
127+
Path location = randomRepoPath();
128+
createRepository(REPO, "fs", location);
129+
130+
{
131+
CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(SYSTEM_DATA_STREAM_NAME);
132+
final AcknowledgedResponse response = client().execute(CreateDataStreamAction.INSTANCE, request).get();
133+
assertTrue(response.isAcknowledged());
134+
}
135+
136+
// Index a doc so that a concrete backing index will be created
137+
IndexResponse indexToDataStreamResponse = client().prepareIndex(SYSTEM_DATA_STREAM_NAME, "_doc")
138+
.setId("42")
139+
.setSource("{ \"@timestamp\": \"2099-03-08T11:06:07.000Z\", \"name\": \"my-name\" }", XContentType.JSON)
140+
.setOpType(DocWriteRequest.OpType.CREATE)
141+
.execute()
142+
.actionGet();
143+
assertThat(indexToDataStreamResponse.status().getStatus(), oneOf(200, 201));
144+
145+
// Index a doc so that a concrete backing index will be created
146+
IndexResponse indexResponse = client().prepareIndex("my-index", "_doc")
147+
.setId("42")
148+
.setSource("{ \"name\": \"my-name\" }", XContentType.JSON)
149+
.setOpType(DocWriteRequest.OpType.CREATE)
150+
.execute()
151+
.get();
152+
assertThat(indexResponse.status().getStatus(), oneOf(200, 201));
153+
154+
{
155+
GetDataStreamAction.Request request = new GetDataStreamAction.Request(new String[] { SYSTEM_DATA_STREAM_NAME });
156+
GetDataStreamAction.Response response = client().execute(GetDataStreamAction.INSTANCE, request).get();
157+
assertThat(response.getDataStreams(), hasSize(1));
158+
assertTrue(response.getDataStreams().get(0).getDataStream().isSystem());
159+
}
160+
161+
SnapshotInfo snapshotInfo = assertSuccessful(
162+
client().admin()
163+
.cluster()
164+
.prepareCreateSnapshot(REPO, SNAPSHOT)
165+
.setIndices("my-index")
166+
.setFeatureStates(SystemDataStreamTestPlugin.class.getSimpleName())
167+
.setWaitForCompletion(true)
168+
.setIncludeGlobalState(false)
169+
.execute()
170+
);
171+
172+
assertThat(snapshotInfo.dataStreams(), not(empty()));
91173

92174
// We have to delete the data stream directly, as the feature reset API doesn't clean up system data streams yet
93175
// See https://github.com/elastic/elasticsearch/issues/75818
@@ -97,6 +179,8 @@ public void testSystemDataStreamSnapshotIT() throws Exception {
97179
assertTrue(response.isAcknowledged());
98180
}
99181

182+
assertAcked(client().admin().indices().prepareDelete("my-index"));
183+
100184
{
101185
GetIndexResponse indicesRemaining = client().admin().indices().prepareGetIndex().addIndices("_all").get();
102186
assertThat(indicesRemaining.indices(), arrayWithSize(0));
@@ -106,7 +190,8 @@ public void testSystemDataStreamSnapshotIT() throws Exception {
106190
.cluster()
107191
.prepareRestoreSnapshot(REPO, SNAPSHOT)
108192
.setWaitForCompletion(true)
109-
.setRestoreGlobalState(false)
193+
.setIndices("my-index")
194+
.setFeatureStates(SystemDataStreamTestPlugin.class.getSimpleName())
110195
.get();
111196
assertEquals(restoreSnapshotResponse.getRestoreInfo().totalShards(), restoreSnapshotResponse.getRestoreInfo().successfulShards());
112197

0 commit comments

Comments
 (0)