From d7f05a1c9e8b2388053ccc19af4837a7755282e2 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 21 Oct 2020 12:54:44 +0200 Subject: [PATCH 01/28] Hidden data streams --- .../indices/GetDataStreamResponseTests.java | 2 +- .../cluster/metadata/DataStream.java | 37 ++++++++++++++++--- .../cluster/metadata/IndexAbstraction.java | 2 +- .../metadata/IndexAbstractionResolver.java | 13 ++----- .../metadata/IndexNameExpressionResolver.java | 4 +- .../MetadataCreateDataStreamService.java | 13 ++++--- .../snapshots/RestoreService.java | 3 +- .../IndexNameExpressionResolverTests.java | 25 +++++++++++++ .../cluster/DataStreamTestHelper.java | 3 +- 9 files changed, 77 insertions(+), 25 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java index 34b5b2af9bf74..efc2e0a8e5a26 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java @@ -55,7 +55,7 @@ private static DataStreamInfo randomInstance() { long generation = indices.size() + randomLongBetween(1, 128); String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(random()))); - DataStream dataStream = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices, generation); + DataStream dataStream = new DataStream(dataStreamName, createTimestampField("@timestamp"), indices, generation, randomBoolean()); return new DataStreamInfo(dataStream, ClusterHealthStatus.YELLOW, randomAlphaOfLengthBetween(2, 10), randomAlphaOfLengthBetween(2, 10)); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 2c3d446306b99..ee7ee118705d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.Version; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.ParseField; @@ -40,17 +41,24 @@ public final class DataStream extends AbstractDiffable implements ToXContentObject { public static final String BACKING_INDEX_PREFIX = ".ds-"; + public static final Version HIDDEN_VERSION = Version.V_8_0_0; private final String name; private final TimestampField timeStampField; private final List indices; private final long generation; + private final boolean hidden; public DataStream(String name, TimestampField timeStampField, List indices, long generation) { + this(name, timeStampField, indices, generation, false); + } + + public DataStream(String name, TimestampField timeStampField, List indices, long generation, boolean hidden) { this.name = name; this.timeStampField = timeStampField; this.indices = Collections.unmodifiableList(indices); this.generation = generation; + this.hidden = hidden; assert indices.size() > 0; assert indices.get(indices.size() - 1).getName().equals(getDefaultBackingIndexName(name, generation)); } @@ -59,6 +67,10 @@ public DataStream(String name, TimestampField timeStampField, List indice this(name, timeStampField, indices, indices.size()); } + public DataStream(String name, TimestampField timeStampField, List indices, boolean hidden) { + this(name, timeStampField, indices, indices.size(), hidden); + } + public String getName() { return name; } @@ -75,6 +87,10 @@ public long getGeneration() { return generation; } + public boolean isHidden() { + return hidden; + } + /** * Performs a rollover on a {@code DataStream} instance and returns a new instance containing * the updated list of backing indices and incremented generation. @@ -87,7 +103,7 @@ public DataStream rollover(Index newWriteIndex) { assert newWriteIndex.getName().equals(getDefaultBackingIndexName(name, generation + 1)); List backingIndices = new ArrayList<>(indices); backingIndices.add(newWriteIndex); - return new DataStream(name, timeStampField, backingIndices, generation + 1); + return new DataStream(name, timeStampField, backingIndices, generation + 1, hidden); } /** @@ -101,7 +117,7 @@ public DataStream removeBackingIndex(Index index) { List backingIndices = new ArrayList<>(indices); backingIndices.remove(index); assert backingIndices.size() == indices.size() - 1; - return new DataStream(name, timeStampField, backingIndices, generation); + return new DataStream(name, timeStampField, backingIndices, generation, hidden); } /** @@ -126,7 +142,7 @@ public DataStream replaceBackingIndex(Index existingBackingIndex, Index newBacki "it is the write index", existingBackingIndex.getName(), name)); } backingIndices.set(backingIndexPosition, newBackingIndex); - return new DataStream(name, timeStampField, backingIndices, generation); + return new DataStream(name, timeStampField, backingIndices, generation, hidden); } /** @@ -142,7 +158,8 @@ public static String getDefaultBackingIndexName(String dataStreamName, long gene } public DataStream(StreamInput in) throws IOException { - this(in.readString(), new TimestampField(in), in.readList(Index::new), in.readVLong()); + this(in.readString(), new TimestampField(in), in.readList(Index::new), in.readVLong(), + in.getVersion().onOrAfter(HIDDEN_VERSION) && in.readBoolean()); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -155,22 +172,31 @@ public void writeTo(StreamOutput out) throws IOException { timeStampField.writeTo(out); out.writeList(indices); out.writeVLong(generation); + if (out.getVersion().onOrAfter(HIDDEN_VERSION)) { + out.writeBoolean(hidden); + } } public static final ParseField NAME_FIELD = new ParseField("name"); public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field"); public static final ParseField INDICES_FIELD = new ParseField("indices"); public static final ParseField GENERATION_FIELD = new ParseField("generation"); + public static final ParseField HIDDEN_FIELD = new ParseField("hidden"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", - args -> new DataStream((String) args[0], (TimestampField) args[1], (List) args[2], (Long) args[3])); + args -> { + Boolean hidden = (Boolean) args[4]; + hidden = hidden != null && hidden; + return new DataStream((String) args[0], (TimestampField) args[1], (List) args[2], (Long) args[3], hidden); + }); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); PARSER.declareObject(ConstructingObjectParser.constructorArg(), TimestampField.PARSER, TIMESTAMP_FIELD_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Index.fromXContent(p), INDICES_FIELD); PARSER.declareLong(ConstructingObjectParser.constructorArg(), GENERATION_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD); } public static DataStream fromXContent(XContentParser parser) throws IOException { @@ -184,6 +210,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(TIMESTAMP_FIELD_FIELD.getPreferredName(), timeStampField); builder.field(INDICES_FIELD.getPreferredName(), indices); builder.field(GENERATION_FIELD.getPreferredName(), generation); + builder.field(HIDDEN_FIELD.getPreferredName(), hidden); builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index 161ada3c74bdd..66e6bb28d210c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -360,7 +360,7 @@ public DataStream getParentDataStream() { @Override public boolean isHidden() { - return false; + return dataStream.isHidden(); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java index da1917f783ab4..9638d3f7a5e88 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java @@ -131,23 +131,18 @@ public static boolean isIndexVisible(String expression, String index, IndicesOpt throw new IllegalStateException("could not resolve index abstraction [" + index + "]"); } final boolean isHidden = indexAbstraction.isHidden(); + boolean isVisible = isHidden == false || indicesOptions.expandWildcardsHidden() || isVisibleDueToImplicitHidden(expression, index); if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) { //it's an alias, ignore expandWildcardsOpen and expandWildcardsClosed. //complicated to support those options with aliases pointing to multiple indices... - if (indicesOptions.ignoreAliases()) { - return false; - } else if (isHidden == false || indicesOptions.expandWildcardsHidden() || isVisibleDueToImplicitHidden(expression, index)) { - return true; - } else { - return false; - } + return isVisible && indicesOptions.ignoreAliases() == false; } if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { - return includeDataStreams; + return isVisible && includeDataStreams; } assert indexAbstraction.getIndices().size() == 1 : "concrete index must point to a single index"; IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0); - if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) { + if (isVisible == false) { return false; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 13063d5486b1d..79ea6201477c1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -260,8 +260,8 @@ Index[] concreteIndices(Context context, String... indexExpressions) { } } else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.includeDataStreams() == false) { - excludedDataStreams = true; - continue; + excludedDataStreams = true; + continue; } if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && context.isResolveToWriteIndex()) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index b47432d202d0e..5ad67937f8ad0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -132,11 +132,12 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn if (request.name.toLowerCase(Locale.ROOT).equals(request.name) == false) { throw new IllegalArgumentException("data_stream [" + request.name + "] must be lowercase"); } - if (request.name.startsWith(".")) { - throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.'"); + if (request.name.startsWith(DataStream.BACKING_INDEX_PREFIX)) { + throw new IllegalArgumentException(String.format("data_stream [%s] must not start with '%s'", request.name, + DataStream.BACKING_INDEX_PREFIX)); } - ComposableIndexTemplate template = lookupTemplateForDataStream(request.name, currentState.metadata()); + ComposableIndexTemplate composableTemplate = lookupTemplateForDataStream(request.name, currentState.metadata()); String firstBackingIndexName = DataStream.getDefaultBackingIndexName(request.name, 1); CreateIndexClusterStateUpdateRequest createIndexRequest = @@ -156,9 +157,11 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn assert firstBackingIndex != null; assert firstBackingIndex.mapping() != null : "no mapping found for backing index [" + firstBackingIndexName + "]"; - String fieldName = template.getDataStreamTemplate().getTimestampField(); + String fieldName = composableTemplate.getDataStreamTemplate().getTimestampField(); DataStream.TimestampField timestampField = new DataStream.TimestampField(fieldName); - DataStream newDataStream = new DataStream(request.name, timestampField, List.of(firstBackingIndex.getIndex())); + Template template = composableTemplate.template(); + boolean hidden = template != null && template.settings() != null && IndexMetadata.INDEX_HIDDEN_SETTING.get(template.settings()); + DataStream newDataStream = new DataStream(request.name, timestampField, List.of(firstBackingIndex.getIndex()), hidden); Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); logger.info("adding data stream [{}]", request.name); return ClusterState.builder(currentState).metadata(builder).build(); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 3821af6861545..35ad25413840c 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -608,7 +608,8 @@ static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metad List updatedIndices = dataStream.getIndices().stream() .map(i -> metadata.get(renameIndex(i.getName(), request, true)).getIndex()) .collect(Collectors.toList()); - return new DataStream(dataStreamName, dataStream.getTimeStampField(), updatedIndices, dataStream.getGeneration()); + return new DataStream(dataStreamName, dataStream.getTimeStampField(), updatedIndices, dataStream.getGeneration(), + dataStream.isHidden()); } public static RestoreInProgress updateRestoreStateWithDeletedIndices(RestoreInProgress oldRestore, Set deletedIndices) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java index ade851cf15f9d..2d984a1107008 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java @@ -2091,6 +2091,31 @@ public void testDataStreamsWithRegularIndexAndAlias() { assertThat(result[2].getName(), equalTo("logs-foobarbaz-0")); } + public void testHiddenDataStreams() { + final String dataStream1 = "logs-foobar"; + IndexMetadata index1 = createBackingIndex(dataStream1, 1).build(); + IndexMetadata index2 = createBackingIndex(dataStream1, 2).build(); + IndexMetadata justAnIndex = IndexMetadata.builder("logs-foobarbaz-0") + .settings(ESTestCase.settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + ClusterState state = ClusterState.builder(new ClusterName("_name")) + .metadata(Metadata.builder() + .put(index1, false) + .put(index2, false) + .put(justAnIndex, false) + .put(new DataStream(dataStream1, createTimestampField("@timestamp"), + List.of(index1.getIndex(), index2.getIndex()),true))).build(); + + Index[] result = indexNameExpressionResolver.concreteIndices(state, IndicesOptions.strictExpandHidden(), true, "logs-*"); + assertThat(result, arrayContainingInAnyOrder(index1.getIndex(), index2.getIndex(), justAnIndex.getIndex() )); + + result = indexNameExpressionResolver.concreteIndices(state, IndicesOptions.strictExpandOpen(), true, "logs-*"); + assertThat(result, arrayContaining(justAnIndex.getIndex())); + } + public void testDataStreamsNames() { final String dataStream1 = "logs-foobar"; final String dataStream2 = "other-foobar"; diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java index c8a7e3f1a8ec0..27bab63d9c997 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java @@ -37,6 +37,7 @@ import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; +import static org.elasticsearch.test.ESTestCase.randomBoolean; public final class DataStreamTestHelper { @@ -103,7 +104,7 @@ public static DataStream randomInstance() { long generation = indices.size() + ESTestCase.randomLongBetween(1, 128); String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(LuceneTestCase.random()))); - return new DataStream(dataStreamName, createTimestampField("@timestamp"), indices, generation); + return new DataStream(dataStreamName, createTimestampField("@timestamp"), indices, generation, randomBoolean()); } /** From b182d368e25d0b4122f311e39ffb74407f7c3045 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 21 Oct 2020 13:00:48 +0200 Subject: [PATCH 02/28] whitespace reverted --- .../cluster/metadata/IndexNameExpressionResolver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 79ea6201477c1..13063d5486b1d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -260,8 +260,8 @@ Index[] concreteIndices(Context context, String... indexExpressions) { } } else if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM && context.includeDataStreams() == false) { - excludedDataStreams = true; - continue; + excludedDataStreams = true; + continue; } if (indexAbstraction.getType() == IndexAbstraction.Type.ALIAS && context.isResolveToWriteIndex()) { From 100dba5f3c9561a2e8bc5c52bffa5881c114b6c7 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 21 Oct 2020 13:27:29 +0200 Subject: [PATCH 03/28] stricter ds name --- .../cluster/metadata/MetadataCreateDataStreamService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 5ad67937f8ad0..3ac7392666532 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -132,9 +132,8 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn if (request.name.toLowerCase(Locale.ROOT).equals(request.name) == false) { throw new IllegalArgumentException("data_stream [" + request.name + "] must be lowercase"); } - if (request.name.startsWith(DataStream.BACKING_INDEX_PREFIX)) { - throw new IllegalArgumentException(String.format("data_stream [%s] must not start with '%s'", request.name, - DataStream.BACKING_INDEX_PREFIX)); + if (request.name.startsWith(".d")) { + throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.d'"); } ComposableIndexTemplate composableTemplate = lookupTemplateForDataStream(request.name, currentState.metadata()); From 0d778f54aa069ed0763122630dcf6d41da48ccd2 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 21 Oct 2020 13:30:04 +0200 Subject: [PATCH 04/28] Revert "stricter ds name" This reverts commit 100dba5f3c9561a2e8bc5c52bffa5881c114b6c7. --- .../cluster/metadata/MetadataCreateDataStreamService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 3ac7392666532..5ad67937f8ad0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -132,8 +132,9 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn if (request.name.toLowerCase(Locale.ROOT).equals(request.name) == false) { throw new IllegalArgumentException("data_stream [" + request.name + "] must be lowercase"); } - if (request.name.startsWith(".d")) { - throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '.d'"); + if (request.name.startsWith(DataStream.BACKING_INDEX_PREFIX)) { + throw new IllegalArgumentException(String.format("data_stream [%s] must not start with '%s'", request.name, + DataStream.BACKING_INDEX_PREFIX)); } ComposableIndexTemplate composableTemplate = lookupTemplateForDataStream(request.name, currentState.metadata()); From 98e45c5229ed6dc52a4ad04f430c3256d8ef2c39 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 21 Oct 2020 13:55:28 +0200 Subject: [PATCH 05/28] String.format removed --- .../cluster/metadata/MetadataCreateDataStreamService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 5ad67937f8ad0..3e9d3497d7c29 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -133,8 +133,8 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn throw new IllegalArgumentException("data_stream [" + request.name + "] must be lowercase"); } if (request.name.startsWith(DataStream.BACKING_INDEX_PREFIX)) { - throw new IllegalArgumentException(String.format("data_stream [%s] must not start with '%s'", request.name, - DataStream.BACKING_INDEX_PREFIX)); + throw new IllegalArgumentException("data_stream [" + request.name + "] must not start with '" + + DataStream.BACKING_INDEX_PREFIX + "'"); } ComposableIndexTemplate composableTemplate = lookupTemplateForDataStream(request.name, currentState.metadata()); From cf74871a0b506ee4264a5ff3e33c27cfb2bbf113 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 21 Oct 2020 17:00:03 +0200 Subject: [PATCH 06/28] fix test --- .../metadata/MetadataCreateDataStreamServiceTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index d52f36afb1fb1..417cdd6858949 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -102,13 +102,13 @@ public void testCreateDataStreamWithUppercaseCharacters() throws Exception { public void testCreateDataStreamStartingWithPeriod() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); - final String dataStreamName = ".may_not_start_with_period"; + final String dataStreamName = ".ds-may_not_start_with_ds"; ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req)); - assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must not start with '.'")); + assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must not start with '.ds-'")); } public void testCreateDataStreamNoTemplate() throws Exception { From a856a8f464f82f397319fbece6c95154fca51c87 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Wed, 21 Oct 2020 23:44:47 +0200 Subject: [PATCH 07/28] fix GetDataStream action --- .../elasticsearch/xpack/core/action/GetDataStreamAction.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java index 0bac97981725b..af927a08d3daa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java @@ -85,7 +85,7 @@ public String[] indices() { public IndicesOptions indicesOptions() { // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and // a data stream's backing indices are retrieved from its metadata - return IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + return IndicesOptions.fromOptions(false, true, true, true, true,false, false, true, false); } @Override @@ -108,6 +108,7 @@ public static class DataStreamInfo extends AbstractDiffable impl public static final ParseField STATUS_FIELD = new ParseField("status"); public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template"); public static final ParseField ILM_POLICY_FIELD = new ParseField("ilm_policy"); + public static final ParseField HIDDEN_FIELD = new ParseField("hidden"); DataStream dataStream; ClusterHealthStatus dataStreamStatus; @@ -166,6 +167,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (ilmPolicyName != null) { builder.field(ILM_POLICY_FIELD.getPreferredName(), ilmPolicyName); } + builder.field(HIDDEN_FIELD.getPreferredName(), dataStream.isHidden()); builder.endObject(); return builder; } From 4a0e20d82c30988faacead6b4eb9b7d018655a11 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 22 Oct 2020 12:25:43 +0200 Subject: [PATCH 08/28] fix test --- .../data-streams/change-mappings-and-settings.asciidoc | 3 ++- docs/reference/data-streams/set-up-a-data-stream.asciidoc | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index 6ff4737ad084e..2b8a6ccf0409d 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -572,7 +572,8 @@ contains information about the stream's oldest backing index, ], "generation": 2, "status": "GREEN", - "template": "my-data-stream-template" + "template": "my-data-stream-template", + "hidden": false } ] } diff --git a/docs/reference/data-streams/set-up-a-data-stream.asciidoc b/docs/reference/data-streams/set-up-a-data-stream.asciidoc index d4feea1d40933..da3df2e9c4516 100644 --- a/docs/reference/data-streams/set-up-a-data-stream.asciidoc +++ b/docs/reference/data-streams/set-up-a-data-stream.asciidoc @@ -217,7 +217,7 @@ You can create a data stream using one of two methods: [[index-documents-to-create-a-data-stream]] ==== Index documents to create a data stream -You can automatically create a data stream using an indexing request. Submit +You can automatically create a data stream using an indexing request. Submit an <> to a target matching the index pattern defined in the template's `index_patterns` property. @@ -352,7 +352,8 @@ contains information about the stream's write index, `.ds-my-data-stream-000002` "generation": 2, "status": "GREEN", "template": "my-data-stream-template", - "ilm_policy": "my-data-stream-policy" + "ilm_policy": "my-data-stream-policy", + "hidden": false } ] } From 19af04d6bdfa19eb5fd802f44ecdb9596adaa45d Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 22 Oct 2020 12:47:02 +0200 Subject: [PATCH 09/28] fix test --- .../elasticsearch/client/indices/DataStream.java | 14 ++++++++++++-- docs/reference/indices/get-data-stream.asciidoc | 3 ++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java index c7d4c61d420cc..8301f9fe7c1fe 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java @@ -36,6 +36,7 @@ public final class DataStream { private final String timeStampField; private final List indices; private final long generation; + private final boolean hidden; ClusterHealthStatus dataStreamStatus; @Nullable String indexTemplate; @@ -43,7 +44,7 @@ public final class DataStream { String ilmPolicyName; public DataStream(String name, String timeStampField, List indices, long generation, ClusterHealthStatus dataStreamStatus, - @Nullable String indexTemplate, @Nullable String ilmPolicyName) { + @Nullable String indexTemplate, @Nullable String ilmPolicyName, boolean hidden) { this.name = name; this.timeStampField = timeStampField; this.indices = indices; @@ -51,6 +52,7 @@ public DataStream(String name, String timeStampField, List indices, long this.dataStreamStatus = dataStreamStatus; this.indexTemplate = indexTemplate; this.ilmPolicyName = ilmPolicyName; + this.hidden = hidden; } public String getName() { @@ -81,6 +83,10 @@ public String getIlmPolicyName() { return ilmPolicyName; } + public boolean isHidden() { + return hidden; + } + public static final ParseField NAME_FIELD = new ParseField("name"); public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field"); public static final ParseField INDICES_FIELD = new ParseField("indices"); @@ -88,6 +94,7 @@ public String getIlmPolicyName() { public static final ParseField STATUS_FIELD = new ParseField("status"); public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template"); public static final ParseField ILM_POLICY_FIELD = new ParseField("ilm_policy"); + public static final ParseField HIDDEN_FIELD = new ParseField("hidden"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", @@ -101,7 +108,9 @@ public String getIlmPolicyName() { ClusterHealthStatus status = ClusterHealthStatus.fromString(statusStr); String indexTemplate = (String) args[5]; String ilmPolicy = (String) args[6]; - return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy); + Boolean hidden = (Boolean) args[7]; + hidden = hidden != null && hidden; + return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, hidden); }); static { @@ -112,6 +121,7 @@ public String getIlmPolicyName() { PARSER.declareString(ConstructingObjectParser.constructorArg(), STATUS_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), INDEX_TEMPLATE_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ILM_POLICY_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD); } public static DataStream fromXContent(XContentParser parser) throws IOException { diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index e25fa3aaf3263..fc465a2fef6ce 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -226,7 +226,8 @@ The API returns the following response: "generation": 1, "status": "YELLOW", "template": "my-index-template", - "ilm_policy": "my-lifecycle-policy" + "ilm_policy": "my-lifecycle-policy", + "hidden": false } ] } From 5b940396e55367fa7013b2f758c2240d68b14d10 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 22 Oct 2020 13:52:09 +0200 Subject: [PATCH 10/28] rest test --- .../indices/get-data-stream.asciidoc | 3 +- .../core/action/DeleteDataStreamAction.java | 2 +- .../plugin/data-streams/qa/rest/build.gradle | 1 + .../datastreams/DataStreamUpgradeRestIT.java | 0 .../xpack/datastreams/DataStreamsRestIT.java | 53 +++++++++++++++++++ .../elasticsearch/xpack/datastreams/a.json | 4 ++ 6 files changed, 61 insertions(+), 2 deletions(-) rename x-pack/plugin/data-streams/qa/rest/src/{yamlRestTest => javaRestTest}/java/org/elasticsearch/xpack/datastreams/DataStreamUpgradeRestIT.java (100%) create mode 100644 x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java create mode 100644 x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/a.json diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index fc465a2fef6ce..61ba4ef63b8aa 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -210,7 +210,8 @@ The API returns the following response: "generation": 2, "status": "GREEN", "template": "my-index-template", - "ilm_policy": "my-lifecycle-policy" + "ilm_policy": "my-lifecycle-policy", + "hidden": false }, { "name": "my-data-stream_two", diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java index 47f64a4b9e1fa..bbf4e93d28af6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java @@ -102,7 +102,7 @@ public String[] indices() { public IndicesOptions indicesOptions() { // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and // a data stream's backing indices are retrieved from its metadata - return IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + return IndicesOptions.fromOptions(false, true, true, true, true,false, false, true, false); } @Override diff --git a/x-pack/plugin/data-streams/qa/rest/build.gradle b/x-pack/plugin/data-streams/qa/rest/build.gradle index 1b65032dd021d..9ddc0858b59e3 100644 --- a/x-pack/plugin/data-streams/qa/rest/build.gradle +++ b/x-pack/plugin/data-streams/qa/rest/build.gradle @@ -1,4 +1,5 @@ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.java-rest-test' restResources { restApi { diff --git a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamUpgradeRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamUpgradeRestIT.java similarity index 100% rename from x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamUpgradeRestIT.java rename to x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamUpgradeRestIT.java diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java new file mode 100644 index 0000000000000..27c5ef6ef2f1e --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +public class DataStreamsRestIT extends ESRestTestCase { + + public void testHiddenDataStream() throws IOException { + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden"); + putComposableIndexTemplateRequest.setJsonEntity("{" + + " \"index_patterns\": [\".hidden\"],\n" + + " \"data_stream\": {},\n" + + " \"template\": {\n" + + " \"settings\": {\n" + + " \"index.hidden\": \"true\"\n" + + " }\n" + + " }\n" + + "}" + ); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + Request createDocRequest = new Request("POST", "/.hidden/_doc"); + createDocRequest.setJsonEntity("{" + + " \"@timestamp\": \"2020-10-22\",\n" + + " \"a\": 1\n" + + "}"); + + assertOK(client().performRequest(createDocRequest)); + + Request getDataStreamsRequest = new Request("GET", "/_data_stream"); + Response response = client().performRequest(getDataStreamsRequest); + Map dataStreams = entityAsMap(response); + assertEquals(Collections.singletonList(".hidden"), XContentMapValues.extractValue("data_streams.name", dataStreams)); + assertEquals(Collections.singletonList("hidden"), XContentMapValues.extractValue("data_streams.template", dataStreams)); + assertEquals(Collections.singletonList(1), XContentMapValues.extractValue("data_streams.generation", dataStreams)); + assertEquals(Collections.singletonList(true), XContentMapValues.extractValue("data_streams.hidden", dataStreams)); + + + } +} diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/a.json b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/a.json new file mode 100644 index 0000000000000..4b8ec46525030 --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/a.json @@ -0,0 +1,4 @@ +{ + "@timestamp": "2020-10-22", + "a": 1 +} From 44415330000762b72aa94ccab65141524f75be54 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 22 Oct 2020 14:09:32 +0200 Subject: [PATCH 11/28] rest test --- .../xpack/datastreams/DataStreamsRestIT.java | 10 +++++++++- ...stIT.java => DataStreamsClientYamlTestSuiteIT.java} | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) rename x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/{DataStreamsRestIT.java => DataStreamsClientYamlTestSuiteIT.java} (79%) diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 27c5ef6ef2f1e..9fb578cc488f6 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -32,7 +32,7 @@ public void testHiddenDataStream() throws IOException { ); assertOK(client().performRequest(putComposableIndexTemplateRequest)); - Request createDocRequest = new Request("POST", "/.hidden/_doc"); + Request createDocRequest = new Request("POST", "/.hidden/_doc?refresh=true"); createDocRequest.setJsonEntity("{" + " \"@timestamp\": \"2020-10-22\",\n" + " \"a\": 1\n" + @@ -48,6 +48,14 @@ public void testHiddenDataStream() throws IOException { assertEquals(Collections.singletonList(1), XContentMapValues.extractValue("data_streams.generation", dataStreams)); assertEquals(Collections.singletonList(true), XContentMapValues.extractValue("data_streams.hidden", dataStreams)); + Request searchRequest = new Request("GET", "/.hidd*/_search"); + response = client().performRequest(searchRequest); + Map results = entityAsMap(response); + assertEquals(0, XContentMapValues.extractValue("hits.total.value", results)); + searchRequest = new Request("GET", "/.hidd*/_search?expand_wildcards=open,hidden"); + response = client().performRequest(searchRequest); + results = entityAsMap(response); + assertEquals(1, XContentMapValues.extractValue("hits.total.value", results)); } } diff --git a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsClientYamlTestSuiteIT.java similarity index 79% rename from x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java rename to x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsClientYamlTestSuiteIT.java index 1d40326391ace..ddca84d5be7cd 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsClientYamlTestSuiteIT.java @@ -10,9 +10,9 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -public class DataStreamsRestIT extends ESClientYamlSuiteTestCase { +public class DataStreamsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { - public DataStreamsRestIT(final ClientYamlTestCandidate testCandidate) { + public DataStreamsClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { super(testCandidate); } From 404d3417b3a38fb52cce82a31540c4eb6a3fc4c0 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 22 Oct 2020 16:51:34 +0200 Subject: [PATCH 12/28] spotless --- .../xpack/datastreams/DataStreamsRestIT.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 9fb578cc488f6..73a047afbbd2f 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -20,23 +20,21 @@ public class DataStreamsRestIT extends ESRestTestCase { public void testHiddenDataStream() throws IOException { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden"); - putComposableIndexTemplateRequest.setJsonEntity("{" + - " \"index_patterns\": [\".hidden\"],\n" + - " \"data_stream\": {},\n" + - " \"template\": {\n" + - " \"settings\": {\n" + - " \"index.hidden\": \"true\"\n" + - " }\n" + - " }\n" + - "}" + putComposableIndexTemplateRequest.setJsonEntity( + "{" + + " \"index_patterns\": [\".hidden\"],\n" + + " \"data_stream\": {},\n" + + " \"template\": {\n" + + " \"settings\": {\n" + + " \"index.hidden\": \"true\"\n" + + " }\n" + + " }\n" + + "}" ); assertOK(client().performRequest(putComposableIndexTemplateRequest)); Request createDocRequest = new Request("POST", "/.hidden/_doc?refresh=true"); - createDocRequest.setJsonEntity("{" + - " \"@timestamp\": \"2020-10-22\",\n" + - " \"a\": 1\n" + - "}"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2020-10-22\", \"a\": 1 }"); assertOK(client().performRequest(createDocRequest)); From 0059e16b4cc949c13c040a3ef2581862fdd5bfd2 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 22 Oct 2020 22:23:46 +0200 Subject: [PATCH 13/28] tests --- .../xpack/datastreams/DataStreamsRestIT.java | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 73a047afbbd2f..cd63ff8629e39 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -22,7 +22,7 @@ public void testHiddenDataStream() throws IOException { Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden"); putComposableIndexTemplateRequest.setJsonEntity( "{" - + " \"index_patterns\": [\".hidden\"],\n" + + " \"index_patterns\": [\"hidden\"],\n" + " \"data_stream\": {},\n" + " \"template\": {\n" + " \"settings\": {\n" @@ -33,7 +33,7 @@ public void testHiddenDataStream() throws IOException { ); assertOK(client().performRequest(putComposableIndexTemplateRequest)); - Request createDocRequest = new Request("POST", "/.hidden/_doc?refresh=true"); + Request createDocRequest = new Request("POST", "/hidden/_doc?refresh=true"); createDocRequest.setJsonEntity("{ \"@timestamp\": \"2020-10-22\", \"a\": 1 }"); assertOK(client().performRequest(createDocRequest)); @@ -41,19 +41,54 @@ public void testHiddenDataStream() throws IOException { Request getDataStreamsRequest = new Request("GET", "/_data_stream"); Response response = client().performRequest(getDataStreamsRequest); Map dataStreams = entityAsMap(response); - assertEquals(Collections.singletonList(".hidden"), XContentMapValues.extractValue("data_streams.name", dataStreams)); + assertEquals(Collections.singletonList("hidden"), XContentMapValues.extractValue("data_streams.name", dataStreams)); assertEquals(Collections.singletonList("hidden"), XContentMapValues.extractValue("data_streams.template", dataStreams)); assertEquals(Collections.singletonList(1), XContentMapValues.extractValue("data_streams.generation", dataStreams)); assertEquals(Collections.singletonList(true), XContentMapValues.extractValue("data_streams.hidden", dataStreams)); - Request searchRequest = new Request("GET", "/.hidd*/_search"); + Request searchRequest = new Request("GET", "/hidd*/_search"); response = client().performRequest(searchRequest); Map results = entityAsMap(response); assertEquals(0, XContentMapValues.extractValue("hits.total.value", results)); - searchRequest = new Request("GET", "/.hidd*/_search?expand_wildcards=open,hidden"); + searchRequest = new Request("GET", "/hidd*/_search?expand_wildcards=open,hidden"); response = client().performRequest(searchRequest); results = entityAsMap(response); assertEquals(1, XContentMapValues.extractValue("hits.total.value", results)); } + + public void testHiddenDataStreamImplicitHiddenSearch() throws IOException { + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden"); + putComposableIndexTemplateRequest.setJsonEntity( + "{" + + " \"index_patterns\": [\".hidden\"],\n" + + " \"data_stream\": {},\n" + + " \"template\": {\n" + + " \"settings\": {\n" + + " \"index.hidden\": \"true\"\n" + + " }\n" + + " }\n" + + "}" + ); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + Request createDocRequest = new Request("POST", "/.hidden/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2020-10-22\", \"a\": 1 }"); + + assertOK(client().performRequest(createDocRequest)); + + Request getDataStreamsRequest = new Request("GET", "/_data_stream"); + Response response = client().performRequest(getDataStreamsRequest); + Map dataStreams = entityAsMap(response); + assertEquals(Collections.singletonList(".hidden"), XContentMapValues.extractValue("data_streams.name", dataStreams)); + assertEquals(Collections.singletonList("hidden"), XContentMapValues.extractValue("data_streams.template", dataStreams)); + assertEquals(Collections.singletonList(1), XContentMapValues.extractValue("data_streams.generation", dataStreams)); + assertEquals(Collections.singletonList(true), XContentMapValues.extractValue("data_streams.hidden", dataStreams)); + + Request searchRequest = new Request("GET", "/.hidd*/_search"); + response = client().performRequest(searchRequest); + Map results = entityAsMap(response); + assertEquals(1, XContentMapValues.extractValue("hits.total.value", results)); + } } From 54dddc4c27590407e731c7cfade5427e33d9fdcb Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Thu, 22 Oct 2020 23:49:14 +0200 Subject: [PATCH 14/28] Delete a.json --- .../java/org/elasticsearch/xpack/datastreams/a.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/a.json diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/a.json b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/a.json deleted file mode 100644 index 4b8ec46525030..0000000000000 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/a.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "@timestamp": "2020-10-22", - "a": 1 -} From 63c0702224915b8186632c57a11976c9776626dc Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Mon, 26 Oct 2020 11:42:26 +0100 Subject: [PATCH 15/28] added expand_wildcards for GetDataStream and DeleteDataStream --- .../core/action/DeleteDataStreamAction.java | 21 ++++++++++++---- .../core/action/GetDataStreamAction.java | 25 +++++++++++++++---- .../rest/RestDeleteDataStreamAction.java | 6 +++-- .../rest/RestGetDataStreamsAction.java | 6 +++-- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java index bbf4e93d28af6..3e8490abecac7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java @@ -25,6 +25,7 @@ public class DeleteDataStreamAction extends ActionType { + public static final Version EXPAND_WILDCARDS_VERSION = Version.V_8_0_0; public static final DeleteDataStreamAction INSTANCE = new DeleteDataStreamAction(); public static final String NAME = "indices:admin/data_stream/delete"; @@ -43,6 +44,7 @@ public static class Request extends MasterNodeRequest implements Indice // empty response can be returned in case wildcards were used or // 404 status code returned in case no wildcard were used. private final boolean wildcardExpressionsOriginallySpecified; + private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); public Request(String[] names) { this.names = Objects.requireNonNull(names); @@ -66,6 +68,9 @@ public Request(StreamInput in) throws IOException { super(in); this.names = in.readStringArray(); this.wildcardExpressionsOriginallySpecified = in.getVersion().onOrAfter(Version.V_7_10_0) && in.readBoolean(); + if (in.getVersion().onOrAfter(EXPAND_WILDCARDS_VERSION)) { + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + } } @Override @@ -75,6 +80,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_10_0)) { out.writeBoolean(wildcardExpressionsOriginallySpecified); } + if(out.getVersion().onOrAfter(EXPAND_WILDCARDS_VERSION)){ + indicesOptions.writeIndicesOptions(out); + } } @Override @@ -83,12 +91,12 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; return wildcardExpressionsOriginallySpecified == request.wildcardExpressionsOriginallySpecified && - Arrays.equals(names, request.names); + Arrays.equals(names, request.names) && indicesOptions.equals(request.indicesOptions); } @Override public int hashCode() { - int result = Objects.hash(wildcardExpressionsOriginallySpecified); + int result = Objects.hash(wildcardExpressionsOriginallySpecified, indicesOptions); result = 31 * result + Arrays.hashCode(names); return result; } @@ -100,9 +108,12 @@ public String[] indices() { @Override public IndicesOptions indicesOptions() { - // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and - // a data stream's backing indices are retrieved from its metadata - return IndicesOptions.fromOptions(false, true, true, true, true,false, false, true, false); + return indicesOptions; + } + + public IndicesRequest indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java index af927a08d3daa..6e6a782a4ae37 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.action; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; @@ -28,6 +29,7 @@ public class GetDataStreamAction extends ActionType { + public static final Version EXPAND_WILDCARDS_VERSION = Version.V_8_0_0; public static final GetDataStreamAction INSTANCE = new GetDataStreamAction(); public static final String NAME = "indices:admin/data_stream/get"; @@ -38,6 +40,7 @@ private GetDataStreamAction() { public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { private String[] names; + private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); public Request(String[] names) { this.names = names; @@ -55,12 +58,18 @@ public ActionRequestValidationException validate() { public Request(StreamInput in) throws IOException { super(in); this.names = in.readOptionalStringArray(); + if(in.getVersion().onOrAfter(EXPAND_WILDCARDS_VERSION)){ + this.indicesOptions = IndicesOptions.readIndicesOptions(in); + } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeOptionalStringArray(names); + if(out.getVersion().onOrAfter(EXPAND_WILDCARDS_VERSION)){ + indicesOptions.writeIndicesOptions(out); + } } @Override @@ -68,12 +77,15 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return Arrays.equals(names, request.names); + return Arrays.equals(names, request.names) && + indicesOptions.equals(request.indicesOptions); } @Override public int hashCode() { - return Arrays.hashCode(names); + int result = Objects.hash(indicesOptions); + result = 31 * result + Arrays.hashCode(names); + return result; } @Override @@ -83,9 +95,12 @@ public String[] indices() { @Override public IndicesOptions indicesOptions() { - // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and - // a data stream's backing indices are retrieved from its metadata - return IndicesOptions.fromOptions(false, true, true, true, true,false, false, true, false); + return indicesOptions; + } + + public Request indicesOptions(IndicesOptions indicesOptions){ + this.indicesOptions = indicesOptions; + return this; } @Override diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java index 1a142453c45c5..f9bffef17bbcf 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java @@ -5,12 +5,13 @@ */ package org.elasticsearch.xpack.datastreams.rest; -import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; import java.io.IOException; import java.util.List; @@ -28,10 +29,11 @@ public List routes() { } @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request( Strings.splitStringByCommaToArray(request.param("name")) ); + deleteDataStreamRequest.indicesOptions(IndicesOptions.fromRequest(request, deleteDataStreamRequest.indicesOptions())); return channel -> client.execute(DeleteDataStreamAction.INSTANCE, deleteDataStreamRequest, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java index 5efdc63bd5f18..f83047975e7b2 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java @@ -5,12 +5,13 @@ */ package org.elasticsearch.xpack.datastreams.rest; -import org.elasticsearch.xpack.core.action.GetDataStreamAction; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.action.GetDataStreamAction; import java.io.IOException; import java.util.List; @@ -28,10 +29,11 @@ public List routes() { } @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { GetDataStreamAction.Request getDataStreamsRequest = new GetDataStreamAction.Request( Strings.splitStringByCommaToArray(request.param("name")) ); + getDataStreamsRequest.indicesOptions(IndicesOptions.fromRequest(request, getDataStreamsRequest.indicesOptions())); return channel -> client.execute(GetDataStreamAction.INSTANCE, getDataStreamsRequest, new RestToXContentListener<>(channel)); } } From dd9afafe7a4771d76059bf306423cd843464f1fe Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Mon, 26 Oct 2020 11:56:46 +0100 Subject: [PATCH 16/28] unused imports --- .../xpack/datastreams/rest/RestDeleteDataStreamAction.java | 1 - .../xpack/datastreams/rest/RestGetDataStreamsAction.java | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java index f9bffef17bbcf..4ceef6c36b619 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; -import java.io.IOException; import java.util.List; public class RestDeleteDataStreamAction extends BaseRestHandler { diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java index f83047975e7b2..0d80a5e6fdd56 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.action.GetDataStreamAction; -import java.io.IOException; import java.util.List; public class RestGetDataStreamsAction extends BaseRestHandler { From 4e71b539b8fe85a442af9d4f0119b97a5b92e707 Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Mon, 26 Oct 2020 23:28:27 +0100 Subject: [PATCH 17/28] add hidden setting to data stream template --- .../metadata/ComposableIndexTemplate.java | 42 ++++++++++++++----- .../cluster/metadata/DataStream.java | 7 +--- .../MetadataCreateDataStreamService.java | 3 +- .../test/rest/ESRestTestCase.java | 2 +- .../core/action/DeleteDataStreamAction.java | 6 +-- .../core/action/GetDataStreamAction.java | 5 +-- .../xpack/datastreams/DataStreamsRestIT.java | 31 +++++--------- 7 files changed, 52 insertions(+), 44 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 28efd3e0593d4..112b079a508cf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -273,20 +272,33 @@ public String toString() { public static class DataStreamTemplate implements Writeable, ToXContentObject { - private static final ObjectParser PARSER = new ObjectParser<>( + private static final ParseField HIDDEN = new ParseField("hidden"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "data_stream_template", - DataStreamTemplate::new - ); + false, + a -> new DataStreamTemplate()); + + static { + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN); + } + + private final boolean hidden; public DataStreamTemplate() { + this(false); } - public String getTimestampField() { - return FIXED_TIMESTAMP_FIELD; + public DataStreamTemplate(boolean hidden) { + this.hidden = hidden; } - DataStreamTemplate(StreamInput in) { - this(); + DataStreamTemplate(StreamInput in) throws IOException { + hidden = in.getVersion().onOrAfter(DataStream.HIDDEN_VERSION) && in.readBoolean(); + } + + public String getTimestampField() { + return FIXED_TIMESTAMP_FIELD; } /** @@ -297,13 +309,21 @@ public Map getDataStreamMappingSnippet() { return Map.of(MapperService.SINGLE_MAPPING_NAME, Map.of("_data_stream_timestamp", Map.of("enabled", true))); } + public boolean isHidden() { + return hidden; + } + @Override public void writeTo(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(DataStream.HIDDEN_VERSION)) { + out.writeBoolean(hidden); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + builder.field("hidden", hidden); builder.endObject(); return builder; } @@ -311,12 +331,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public boolean equals(Object o) { if (this == o) return true; - return o != null && getClass() == o.getClass(); + if (o == null || getClass() != o.getClass()) return false; + DataStreamTemplate that = (DataStreamTemplate) o; + return hidden == that.hidden; } @Override public int hashCode() { - return DataStreamTemplate.class.hashCode(); + return Objects.hash(hidden); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index ee7ee118705d4..b70f756c87913 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -185,11 +185,8 @@ public void writeTo(StreamOutput out) throws IOException { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", - args -> { - Boolean hidden = (Boolean) args[4]; - hidden = hidden != null && hidden; - return new DataStream((String) args[0], (TimestampField) args[1], (List) args[2], (Long) args[3], hidden); - }); + args -> new DataStream((String) args[0], (TimestampField) args[1], (List) args[2], (Long) args[3], + args[4] != null && (boolean) args[4])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 3e9d3497d7c29..248d6dc0f7343 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -159,8 +159,7 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn String fieldName = composableTemplate.getDataStreamTemplate().getTimestampField(); DataStream.TimestampField timestampField = new DataStream.TimestampField(fieldName); - Template template = composableTemplate.template(); - boolean hidden = template != null && template.settings() != null && IndexMetadata.INDEX_HIDDEN_SETTING.get(template.settings()); + boolean hidden = composableTemplate.getDataStreamTemplate().isHidden(); DataStream newDataStream = new DataStream(request.name, timestampField, List.of(firstBackingIndex.getIndex()), hidden); Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); logger.info("adding data stream [{}]", request.name); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index ec81c4c268319..6a37e1d40bbd2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -677,7 +677,7 @@ protected static void wipeAllIndices() throws IOException { protected static void wipeDataStreams() throws IOException { try { if (hasXPack()) { - adminClient().performRequest(new Request("DELETE", "_data_stream/*")); + adminClient().performRequest(new Request("DELETE", "_data_stream/*?expand_wildcards=all")); } } catch (ResponseException e) { // We hit a version of ES that doesn't serialize DeleteDataStreamAction.Request#wildcardExpressionsOriginallySpecified field or diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java index 3e8490abecac7..23cc8e2f2559f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; @@ -25,7 +26,6 @@ public class DeleteDataStreamAction extends ActionType { - public static final Version EXPAND_WILDCARDS_VERSION = Version.V_8_0_0; public static final DeleteDataStreamAction INSTANCE = new DeleteDataStreamAction(); public static final String NAME = "indices:admin/data_stream/delete"; @@ -68,7 +68,7 @@ public Request(StreamInput in) throws IOException { super(in); this.names = in.readStringArray(); this.wildcardExpressionsOriginallySpecified = in.getVersion().onOrAfter(Version.V_7_10_0) && in.readBoolean(); - if (in.getVersion().onOrAfter(EXPAND_WILDCARDS_VERSION)) { + if (in.getVersion().onOrAfter(DataStream.HIDDEN_VERSION)) { this.indicesOptions = IndicesOptions.readIndicesOptions(in); } } @@ -80,7 +80,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_10_0)) { out.writeBoolean(wildcardExpressionsOriginallySpecified); } - if(out.getVersion().onOrAfter(EXPAND_WILDCARDS_VERSION)){ + if (out.getVersion().onOrAfter(DataStream.HIDDEN_VERSION)) { indicesOptions.writeIndicesOptions(out); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java index 6e6a782a4ae37..b40213833be33 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java @@ -29,7 +29,6 @@ public class GetDataStreamAction extends ActionType { - public static final Version EXPAND_WILDCARDS_VERSION = Version.V_8_0_0; public static final GetDataStreamAction INSTANCE = new GetDataStreamAction(); public static final String NAME = "indices:admin/data_stream/get"; @@ -58,7 +57,7 @@ public ActionRequestValidationException validate() { public Request(StreamInput in) throws IOException { super(in); this.names = in.readOptionalStringArray(); - if(in.getVersion().onOrAfter(EXPAND_WILDCARDS_VERSION)){ + if (in.getVersion().onOrAfter(DataStream.HIDDEN_VERSION)) { this.indicesOptions = IndicesOptions.readIndicesOptions(in); } } @@ -67,7 +66,7 @@ public Request(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeOptionalStringArray(names); - if(out.getVersion().onOrAfter(EXPAND_WILDCARDS_VERSION)){ + if (out.getVersion().onOrAfter(DataStream.HIDDEN_VERSION)) { indicesOptions.writeIndicesOptions(out); } } diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index cd63ff8629e39..d5e44875e6987 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -17,19 +17,15 @@ public class DataStreamsRestIT extends ESRestTestCase { + public void testHiddenDataStream() throws IOException { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden"); putComposableIndexTemplateRequest.setJsonEntity( - "{" - + " \"index_patterns\": [\"hidden\"],\n" - + " \"data_stream\": {},\n" - + " \"template\": {\n" - + " \"settings\": {\n" - + " \"index.hidden\": \"true\"\n" - + " }\n" - + " }\n" - + "}" + "{\n" + + " \"index_patterns\": [ \"hidden\" ],\n" + + " \"data_stream\": { \"hidden\": true }\n" + + "}" ); assertOK(client().performRequest(putComposableIndexTemplateRequest)); @@ -38,7 +34,7 @@ public void testHiddenDataStream() throws IOException { assertOK(client().performRequest(createDocRequest)); - Request getDataStreamsRequest = new Request("GET", "/_data_stream"); + Request getDataStreamsRequest = new Request("GET", "/_data_stream?expand_wildcards=all"); Response response = client().performRequest(getDataStreamsRequest); Map dataStreams = entityAsMap(response); assertEquals(Collections.singletonList("hidden"), XContentMapValues.extractValue("data_streams.name", dataStreams)); @@ -61,15 +57,10 @@ public void testHiddenDataStreamImplicitHiddenSearch() throws IOException { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden"); putComposableIndexTemplateRequest.setJsonEntity( - "{" - + " \"index_patterns\": [\".hidden\"],\n" - + " \"data_stream\": {},\n" - + " \"template\": {\n" - + " \"settings\": {\n" - + " \"index.hidden\": \"true\"\n" - + " }\n" - + " }\n" - + "}" + "{\n" + + " \"index_patterns\": [ \".hidden\" ],\n" + + " \"data_stream\": { \"hidden\": true }\n" + + "}" ); assertOK(client().performRequest(putComposableIndexTemplateRequest)); @@ -78,7 +69,7 @@ public void testHiddenDataStreamImplicitHiddenSearch() throws IOException { assertOK(client().performRequest(createDocRequest)); - Request getDataStreamsRequest = new Request("GET", "/_data_stream"); + Request getDataStreamsRequest = new Request("GET", "/_data_stream?expand_wildcards=all"); Response response = client().performRequest(getDataStreamsRequest); Map dataStreams = entityAsMap(response); assertEquals(Collections.singletonList(".hidden"), XContentMapValues.extractValue("data_streams.name", dataStreams)); From f00df2d4b67b524bf6a299b04cdc6654652d42ea Mon Sep 17 00:00:00 2001 From: Przemko Robakowski Date: Mon, 26 Oct 2020 23:55:31 +0100 Subject: [PATCH 18/28] fix expand_wildcards --- .../action/support/IndicesOptions.java | 7 +++++++ .../metadata/ComposableIndexTemplate.java | 2 +- .../xpack/datastreams/DataStreamsRestIT.java | 19 ++++--------------- .../DeleteDataStreamTransportAction.java | 8 +++++++- .../action/GetDataStreamsTransportAction.java | 8 +++++++- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index e68c0f271e135..65fb2b75d693f 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -225,6 +225,13 @@ public EnumSet getExpandWildcards() { return EnumSet.copyOf(expandWildcards); } + /** + * @return a copy of the {@link Option}s that these indices options will use + */ + public EnumSet