From c552bc89ebe9d51240e15eeb58ba917fbdc05624 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 20 Sep 2021 11:35:54 -0400 Subject: [PATCH 1/7] TSDB: Automatically map timestamp If tsdb is enabled we need an `@timestamp` field. This automatically maps the field if it is missing and fails to create indices in time_series mode that map `@timestamp` as anything other than `date` and `date_nanos`. (cherry picked from commit 59694d5c481321efcdfa6fb888c025a0170a5d22) --- .../test/tsdb/15_timestamp_mapping.yml | 151 ++++++++++++++++++ .../org/elasticsearch/index/IndexMode.java | 34 ++++ .../index/mapper/ObjectMapper.java | 5 + .../index/mapper/RootObjectMapper.java | 1 + .../index/TimeSeriesModeTests.java | 50 +++++- 5 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml new file mode 100644 index 0000000000000..88002fc6c3228 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -0,0 +1,151 @@ + +--- +date: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + dimension: true + + - do: + indices.get_mapping: + index: test + - match: { "test.mappings.properties.@timestamp.type": date } + + - do: + bulk: + refresh: true + index: test_index + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}' + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: {hits.total.value: 1} + - match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] } + +--- +date_nanos: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date_nanos + metricset: + type: keyword + dimension: true + + - do: + indices.get_mapping: + index: test + - match: { "test.mappings.properties.@timestamp.type": date_nanos } + + - do: + bulk: + refresh: true + index: test_index + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}' + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: {hits.total.value: 1} + - match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] } + +--- +automatically add with date: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + metricset: + type: keyword + dimension: true + + - do: + indices.get_mapping: + index: test + - match: { 'test.mappings.properties.@timestamp': { "type": date } } + + - do: + bulk: + refresh: true + index: test_index + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}' + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: {hits.total.value: 1} + - match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] } + +--- +reject @timestamp with wrong type: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + catch: /@timestamp must be \[date\] or \[date_nanos\]/ + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: keyword diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index c92a5d742ec09..56fffb09cab04 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -11,10 +11,15 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MappingParserContext; +import org.elasticsearch.index.mapper.RootObjectMapper; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.stream.Stream; import static java.util.stream.Collectors.toSet; @@ -26,6 +31,9 @@ public enum IndexMode { STANDARD { @Override void validateWithOtherSettings(Map, Object> settings) {} + + @Override + public void completeMappings(MappingParserContext context, RootObjectMapper.Builder builder) {} }, TIME_SERIES { @Override @@ -43,6 +51,27 @@ void validateWithOtherSettings(Map, Object> settings) { private String error(Setting unsupported) { return "[" + IndexSettings.MODE.getKey() + "=time_series] is incompatible with [" + unsupported.getKey() + "]"; } + + @Override + public void completeMappings(MappingParserContext context, RootObjectMapper.Builder builder) { + Optional timestamp = builder.getBuilder("@timestamp"); + if (timestamp.isEmpty()) { + builder.add( + new DateFieldMapper.Builder( + "@timestamp", + DateFieldMapper.Resolution.MILLISECONDS, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + context.scriptCompiler(), + DateFieldMapper.IGNORE_MALFORMED_SETTING.get(context.getSettings()), + context.getIndexSettings().getIndexVersionCreated() + ) + ); + return; + } + if (false == timestamp.get() instanceof DateFieldMapper.Builder) { + throw new IllegalArgumentException("@timestamp must be [date] or [date_nanos]"); + } + } }; private static final List> TIME_SERIES_UNSUPPORTED = List.of( @@ -57,4 +86,9 @@ private String error(Setting unsupported) { ); abstract void validateWithOtherSettings(Map, Object> settings); + + /** + * Validate and/or modify the mappings after after they've been parsed. + */ + public abstract void completeMappings(MappingParserContext context, RootObjectMapper.Builder builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 524758243ca58..afa6a338c5273 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; public class ObjectMapper extends Mapper implements Cloneable { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ObjectMapper.class); @@ -86,6 +87,10 @@ public Builder add(Mapper.Builder builder) { return this; } + public Optional getBuilder(String name) { + return mappersBuilders.stream().filter(b -> b.name().equals(name)).findFirst(); + } + protected final Map buildMappers(boolean root, MapperBuilderContext context) { if (root == false) { context = context.createChildContext(name); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index ed2e51660d343..0786455439a68 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -151,6 +151,7 @@ public RootObjectMapper.Builder parse(String name, Map node, Map iterator.remove(); } } + parserContext.getIndexSettings().getMode().completeMappings(parserContext, builder); return builder; } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index c0e73bc833b42..5de7e8d1def9a 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -10,11 +10,19 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperServiceTestCase; + +import java.io.IOException; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; -public class TimeSeriesModeTests extends ESTestCase { +public class TimeSeriesModeTests extends MapperServiceTestCase { public void testPartitioned() { Settings s = Settings.builder() .put(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.getKey(), 2) @@ -50,4 +58,42 @@ public void testSortOrder() { Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s)); assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); } + + public void testAddsTimestamp() throws IOException { + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> {})).documentMapper(); + MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); + assertThat(timestamp, instanceOf(DateFieldType.class)); + assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); + } + + public void testTimestampMillis() throws IOException { + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) + .documentMapper(); + MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); + assertThat(timestamp, instanceOf(DateFieldType.class)); + assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); + } + + public void testTimestampNanos() throws IOException { + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date_nanos").endObject())) + .documentMapper(); + MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); + assertThat(timestamp, instanceOf(DateFieldType.class)); + assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.NANOSECONDS)); + } + + public void testBadTimestamp() throws IOException { + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + Exception e = expectThrows( + MapperParsingException.class, + () -> createMapperService( + s, + mapping(b -> b.startObject("@timestamp").field("type", randomFrom("keyword", "int", "long", "double", "text")).endObject()) + ) + ); + assertThat(e.getMessage(), equalTo("Failed to parse mapping: @timestamp must be [date] or [date_nanos]")); + } } From 4d38bdb65fd1ac5f7f64443cbd1ce473cc5f35b5 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 14 Oct 2021 17:37:51 +0800 Subject: [PATCH 2/7] add timestamp metafield to time_series index --- .../test/tsdb/15_timestamp_mapping.yml | 2 +- .../org/elasticsearch/index/IndexMode.java | 35 ++++++++++++++----- .../DataStreamTimestampFieldMapper.java | 4 +-- .../index/mapper/MappingParser.java | 13 ++++--- .../index/mapper/RootObjectMapper.java | 1 - .../elasticsearch/indices/IndicesModule.java | 2 ++ .../index/TimeSeriesModeTests.java | 27 +++++++++++--- .../xpack/datastreams/DataStreamsPlugin.java | 11 +----- 8 files changed, 63 insertions(+), 32 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml index 88002fc6c3228..1b22340a95864 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -136,7 +136,7 @@ reject @timestamp with wrong type: reason: introduced in 8.0.0 to be backported to 7.16.0 - do: - catch: /@timestamp must be \[date\] or \[date_nanos\]/ + catch: /data stream timestamp field \[@timestamp\] is of type keyword, but \[date,date_nanos\] is expected/ indices.create: index: test body: diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 56fffb09cab04..1f0efe762dc48 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -11,9 +11,11 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MappingParserContext; +import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.RootObjectMapper; import java.util.List; @@ -33,7 +35,11 @@ public enum IndexMode { void validateWithOtherSettings(Map, Object> settings) {} @Override - public void completeMappings(MappingParserContext context, RootObjectMapper.Builder builder) {} + public void completeMappings( + MappingParserContext context, + Map, MetadataFieldMapper> metadataMappers, + RootObjectMapper.Builder builder + ) {} }, TIME_SERIES { @Override @@ -53,12 +59,23 @@ private String error(Setting unsupported) { } @Override - public void completeMappings(MappingParserContext context, RootObjectMapper.Builder builder) { - Optional timestamp = builder.getBuilder("@timestamp"); + public void completeMappings( + MappingParserContext context, + Map, MetadataFieldMapper> metadataMappers, + RootObjectMapper.Builder builder + ) { + DataStreamTimestampFieldMapper timestampFieldMapper = (DataStreamTimestampFieldMapper) metadataMappers.get( + DataStreamTimestampFieldMapper.class + ); + if (timestampFieldMapper == null || false == timestampFieldMapper.isEnabled()) { + metadataMappers.put(DataStreamTimestampFieldMapper.class, DataStreamTimestampFieldMapper.ENABLED_INSTANCE); + } + + Optional timestamp = builder.getBuilder(DataStreamTimestampFieldMapper.DEFAULT_PATH); if (timestamp.isEmpty()) { builder.add( new DateFieldMapper.Builder( - "@timestamp", + DataStreamTimestampFieldMapper.DEFAULT_PATH, DateFieldMapper.Resolution.MILLISECONDS, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, context.scriptCompiler(), @@ -66,10 +83,6 @@ public void completeMappings(MappingParserContext context, RootObjectMapper.Buil context.getIndexSettings().getIndexVersionCreated() ) ); - return; - } - if (false == timestamp.get() instanceof DateFieldMapper.Builder) { - throw new IllegalArgumentException("@timestamp must be [date] or [date_nanos]"); } } }; @@ -90,5 +103,9 @@ public void completeMappings(MappingParserContext context, RootObjectMapper.Buil /** * Validate and/or modify the mappings after after they've been parsed. */ - public abstract void completeMappings(MappingParserContext context, RootObjectMapper.Builder builder); + public abstract void completeMappings( + MappingParserContext context, + Map, MetadataFieldMapper> metadataMappers, + RootObjectMapper.Builder builder + ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 88b8f6305e354..616c10b052408 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -32,9 +32,9 @@ public class DataStreamTimestampFieldMapper extends MetadataFieldMapper { public static final String NAME = "_data_stream_timestamp"; - private static final String DEFAULT_PATH = "@timestamp"; + public static final String DEFAULT_PATH = "@timestamp"; - private static final DataStreamTimestampFieldMapper ENABLED_INSTANCE = + public static final DataStreamTimestampFieldMapper ENABLED_INSTANCE = new DataStreamTimestampFieldMapper(TimestampFieldType.INSTANCE, true); private static final DataStreamTimestampFieldMapper DISABLED_INSTANCE = new DataStreamTimestampFieldMapper(TimestampFieldType.INSTANCE, false); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 727e7622c27df..819642476e8e1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -10,8 +10,8 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.XContentType; import java.util.Collections; import java.util.HashMap; @@ -94,8 +94,11 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar private Mapping parse(String type, Map mapping) throws MapperParsingException { MappingParserContext parserContext = parserContextSupplier.get(); - RootObjectMapper rootObjectMapper - = rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT); + + // parserContext.getIndexSettings().getMode().completeMappings(parserContext, mapping); + + RootObjectMapper.Builder rootObjectMapperBuilder + = rootObjectTypeParser.parse(type, mapping, parserContext); Map, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get(); Map meta = null; @@ -143,8 +146,10 @@ private Mapping parse(String type, Map mapping) throws MapperPar } checkNoRemainingFields(mapping, "Root mapping definition has unsupported parameters: "); + parserContext.getIndexSettings().getMode().completeMappings(parserContext, metadataMappers, rootObjectMapperBuilder); + return new Mapping( - rootObjectMapper, + rootObjectMapperBuilder.build(MapperBuilderContext.ROOT), metadataMappers.values().toArray(new MetadataFieldMapper[0]), meta); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 0786455439a68..ed2e51660d343 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -151,7 +151,6 @@ public RootObjectMapper.Builder parse(String name, Map node, Map iterator.remove(); } } - parserContext.getIndexSettings().getMode().completeMappings(parserContext, builder); return builder; } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 37896dc1edb46..9cf04054bd92d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.rollover.MaxPrimaryShardSizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.resync.TransportResyncReplicationAction; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -183,6 +184,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(VersionFieldMapper.NAME, VersionFieldMapper.PARSER); builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER); builtInMetadataMappers.put(DocCountFieldMapper.NAME, DocCountFieldMapper.PARSER); + builtInMetadataMappers.put(DataStreamTimestampFieldMapper.NAME, DataStreamTimestampFieldMapper.PARSER); //_field_names must be added last so that it has a chance to see all the other mappers builtInMetadataMappers.put(FieldNamesFieldMapper.NAME, FieldNamesFieldMapper.PARSER); return Collections.unmodifiableMap(builtInMetadataMappers); diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 5de7e8d1def9a..173e5c71bc66d 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -10,11 +10,12 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperServiceTestCase; import java.io.IOException; @@ -62,9 +63,13 @@ public void testSortOrder() { public void testAddsTimestamp() throws IOException { Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); DocumentMapper mapper = createMapperService(s, mapping(b -> {})).documentMapper(); - MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); + MappedFieldType timestamp = mapper.mappers().getFieldType(DataStreamTimestampFieldMapper.DEFAULT_PATH); assertThat(timestamp, instanceOf(DateFieldType.class)); assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); + + Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); + assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); + assertTrue(((DataStreamTimestampFieldMapper)timestampField).isEnabled()); } public void testTimestampMillis() throws IOException { @@ -74,6 +79,10 @@ public void testTimestampMillis() throws IOException { MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); assertThat(timestamp, instanceOf(DateFieldType.class)); assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); + + Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); + assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); + assertTrue(((DataStreamTimestampFieldMapper)timestampField).isEnabled()); } public void testTimestampNanos() throws IOException { @@ -83,17 +92,25 @@ public void testTimestampNanos() throws IOException { MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); assertThat(timestamp, instanceOf(DateFieldType.class)); assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.NANOSECONDS)); + + Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); + assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); + assertTrue(((DataStreamTimestampFieldMapper)timestampField).isEnabled()); } public void testBadTimestamp() throws IOException { Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + String type = randomFrom("keyword", "integer", "long", "double", "text"); Exception e = expectThrows( - MapperParsingException.class, + IllegalArgumentException.class, () -> createMapperService( s, - mapping(b -> b.startObject("@timestamp").field("type", randomFrom("keyword", "int", "long", "double", "text")).endObject()) + mapping(b -> b.startObject("@timestamp").field("type", type).endObject()) ) ); - assertThat(e.getMessage(), equalTo("Failed to parse mapping: @timestamp must be [date] or [date_nanos]")); + assertThat( + e.getMessage(), + equalTo("data stream timestamp field [@timestamp] is of type [" + type + "], but [date,date_nanos] is expected") + ); } } diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java index 3c5e954e68ce6..2aae909fea0e5 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java @@ -14,10 +14,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; -import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.plugins.ActionPlugin; -import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; @@ -45,15 +42,9 @@ import org.elasticsearch.xpack.datastreams.rest.RestPromoteDataStreamAction; import java.util.List; -import java.util.Map; import java.util.function.Supplier; -public class DataStreamsPlugin extends Plugin implements ActionPlugin, MapperPlugin { - - @Override - public Map getMetadataMappers() { - return Map.of(DataStreamTimestampFieldMapper.NAME, DataStreamTimestampFieldMapper.PARSER); - } +public class DataStreamsPlugin extends Plugin implements ActionPlugin { @Override public List> getActions() { From 986b0a81fbb9eb2f63b2330bdf5c080685d467d4 Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 18 Oct 2021 16:27:12 +0800 Subject: [PATCH 3/7] add timestamp meta field to time_series index --- .../test/tsdb/15_timestamp_mapping.yml | 36 ++++++- .../org/elasticsearch/index/IndexMode.java | 47 +++++----- .../index/mapper/MappingParser.java | 6 +- .../MetadataDataStreamsServiceTests.java | 7 -- .../MetadataIndexTemplateServiceTests.java | 30 +----- ...tadataMigrateToDataStreamServiceTests.java | 7 -- .../index/TimeSeriesModeTests.java | 93 ++++++++++++++++--- .../index/mapper/DocumentMapperTests.java | 6 +- .../index/mapper/MapperServiceTests.java | 8 +- .../indices/IndicesModuleTests.java | 17 +++- 10 files changed, 167 insertions(+), 90 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml index 1b22340a95864..9d61d4c359b6d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -12,6 +12,7 @@ date: settings: index: mode: time_series + routing_path: [metricset] number_of_replicas: 0 number_of_shards: 2 mappings: @@ -20,12 +21,13 @@ date: type: date metricset: type: keyword - dimension: true + time_series_dimension: true - do: indices.get_mapping: index: test - match: { "test.mappings.properties.@timestamp.type": date } + - match: { 'test.mappings._data_stream_timestamp.enabled': true } - do: bulk: @@ -56,6 +58,7 @@ date_nanos: settings: index: mode: time_series + routing_path: [metricset] number_of_replicas: 0 number_of_shards: 2 mappings: @@ -64,12 +67,13 @@ date_nanos: type: date_nanos metricset: type: keyword - dimension: true + time_series_dimension: true - do: indices.get_mapping: index: test - match: { "test.mappings.properties.@timestamp.type": date_nanos } + - match: { 'test.mappings._data_stream_timestamp.enabled': true } - do: bulk: @@ -100,18 +104,20 @@ automatically add with date: settings: index: mode: time_series + routing_path: [metricset] number_of_replicas: 0 number_of_shards: 2 mappings: properties: metricset: type: keyword - dimension: true + time_series_dimension: true - do: indices.get_mapping: index: test - match: { 'test.mappings.properties.@timestamp': { "type": date } } + - match: { 'test.mappings._data_stream_timestamp.enabled': true } - do: bulk: @@ -136,16 +142,38 @@ reject @timestamp with wrong type: reason: introduced in 8.0.0 to be backported to 7.16.0 - do: - catch: /data stream timestamp field \[@timestamp\] is of type keyword, but \[date,date_nanos\] is expected/ + catch: /data stream timestamp field \[@timestamp\] is of type \[keyword\], but \[date,date_nanos\] is expected/ indices.create: index: test body: settings: index: mode: time_series + routing_path: [metricset] number_of_replicas: 0 number_of_shards: 2 mappings: properties: "@timestamp": type: keyword + +--- +reject timestamp meta field with wrong type: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + catch: /.* time series index \[_data_stream_timestamp\] meta field must be enabled/ + indices.create: + index: test + body: + settings: + index: + mode: time_series + routing_path: [metricset] + number_of_replicas: 0 + number_of_shards: 2 + mappings: + _data_stream_timestamp: + enabled: false diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 91d2f4139189d..1e46b21c60d70 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -11,16 +11,17 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; -import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -52,11 +53,7 @@ void validateWithOtherSettings(Map, Object> settings) { public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) {} @Override - public void completeMappings( - MappingParserContext context, - Map, MetadataFieldMapper> metadataMappers, - RootObjectMapper.Builder builder - ) {} + public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) {} }, TIME_SERIES { @Override @@ -102,16 +99,11 @@ private String tsdbMode() { } @Override - public void completeMappings( - MappingParserContext context, - Map, MetadataFieldMapper> metadataMappers, - RootObjectMapper.Builder builder - ) { - DataStreamTimestampFieldMapper timestampFieldMapper = (DataStreamTimestampFieldMapper) metadataMappers.get( - DataStreamTimestampFieldMapper.class - ); - if (timestampFieldMapper == null || false == timestampFieldMapper.isEnabled()) { - metadataMappers.put(DataStreamTimestampFieldMapper.class, DataStreamTimestampFieldMapper.ENABLED_INSTANCE); + public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) { + if (false == mapping.containsKey(DataStreamTimestampFieldMapper.NAME)) { + mapping.put(DataStreamTimestampFieldMapper.NAME, new HashMap<>(Map.of("enabled", true))); + } else { + validateTimeStampField(mapping.get(DataStreamTimestampFieldMapper.NAME)); } Optional timestamp = builder.getBuilder(DataStreamTimestampFieldMapper.DEFAULT_PATH); @@ -128,6 +120,23 @@ public void completeMappings( ); } } + + private void validateTimeStampField(Object timestampFieldValue) { + if (false == (timestampFieldValue instanceof Map)) { + throw new IllegalArgumentException( + "time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field format error" + ); + } + + @SuppressWarnings("unchecked") + Map timeStampFieldValueMap = (Map) timestampFieldValue; + if (false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", true)) + && false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", "true"))) { + throw new IllegalArgumentException( + "time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field must be enabled" + ); + } + } }; private static final List> TIME_SERIES_UNSUPPORTED = List.of( @@ -159,9 +168,5 @@ public void completeMappings( /** * Validate and/or modify the mappings after after they've been parsed. */ - public abstract void completeMappings( - MappingParserContext context, - Map, MetadataFieldMapper> metadataMappers, - RootObjectMapper.Builder builder - ); + public abstract void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 819642476e8e1..f9204c51608c2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -94,11 +94,9 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar private Mapping parse(String type, Map mapping) throws MapperParsingException { MappingParserContext parserContext = parserContextSupplier.get(); - - // parserContext.getIndexSettings().getMode().completeMappings(parserContext, mapping); - RootObjectMapper.Builder rootObjectMapperBuilder = rootObjectTypeParser.parse(type, mapping, parserContext); + parserContext.getIndexSettings().getMode().completeMappings(parserContext, mapping, rootObjectMapperBuilder); Map, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get(); Map meta = null; @@ -146,8 +144,6 @@ private Mapping parse(String type, Map mapping) throws MapperPar } checkNoRemainingFields(mapping, "Root mapping definition has unsupported parameters: "); - parserContext.getIndexSettings().getMode().completeMappings(parserContext, metadataMappers, rootObjectMapperBuilder); - return new Mapping( rootObjectMapperBuilder.build(MapperBuilderContext.ROOT), metadataMappers.values().toArray(new MetadataFieldMapper[0]), diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java index 92206ce8e5ea5..58f6c443722f2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java @@ -16,11 +16,9 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; -import org.elasticsearch.plugins.Plugin; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -352,9 +350,4 @@ private MapperService getMapperService(IndexMetadata im) { throw new IllegalStateException(e); } } - - @Override - protected Collection getPlugins() { - return List.of(new MetadataIndexTemplateServiceTests.DummyPlugin()); - } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index f9f623979f5af..f66194fe9ef57 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.metadata; import com.fasterxml.jackson.core.JsonParseException; + import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -23,27 +24,22 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.test.ClusterServiceUtils; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexTemplateException; -import org.elasticsearch.plugins.MapperPlugin; -import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentFactory; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -78,11 +74,6 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { - @Override - protected Collection> getPlugins() { - return List.of(DummyPlugin.class); - } - public void testLegacyNoopUpdate() { ClusterState state = ClusterState.EMPTY_STATE; PutRequest pr = new PutRequest("api", "id"); @@ -1595,15 +1586,4 @@ clusterService, createIndexService, new AliasValidator(), indicesService, public static void assertTemplatesEqual(ComposableIndexTemplate actual, ComposableIndexTemplate expected) { assertTrue(Objects.equals(actual, expected)); } - - // Composable index template with data_stream definition need _timestamp meta field mapper, - // this is a dummy impl, so that tests don't fail with the fact that the _timestamp field can't be found. - // (tests using this dummy impl doesn't test the _timestamp validation, but need it to tests other functionality) - public static class DummyPlugin extends Plugin implements MapperPlugin { - - @Override - public Map getMetadataMappers() { - return Map.of(DataStreamTimestampFieldMapper.NAME, DataStreamTimestampFieldMapper.PARSER); - } - } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index d89825ddecf96..7f394801c8106 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -18,10 +18,8 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.indices.EmptySystemIndices; -import org.elasticsearch.plugins.Plugin; import java.io.IOException; -import java.util.Collection; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -326,9 +324,4 @@ private MetadataCreateIndexService getMetadataCreateIndexService() { when(service.getSystemIndices()).thenReturn(EmptySystemIndices.INSTANCE); return service; } - - @Override - protected Collection getPlugins() { - return List.of(new MetadataIndexTemplateServiceTests.DummyPlugin()); - } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 3d8b9e9b045f5..cea2f99a50e06 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -16,7 +16,10 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; @@ -70,7 +73,10 @@ public void testSortOrder() { } public void testAddsTimestamp() throws IOException { - Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); DocumentMapper mapper = createMapperService(s, mapping(b -> {})).documentMapper(); MappedFieldType timestamp = mapper.mappers().getFieldType(DataStreamTimestampFieldMapper.DEFAULT_PATH); assertThat(timestamp, instanceOf(DateFieldType.class)); @@ -78,11 +84,14 @@ public void testAddsTimestamp() throws IOException { Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper)timestampField).isEnabled()); + assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); } public void testTimestampMillis() throws IOException { - Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) .documentMapper(); MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); @@ -91,11 +100,14 @@ public void testTimestampMillis() throws IOException { Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper)timestampField).isEnabled()); + assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); } public void testTimestampNanos() throws IOException { - Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date_nanos").endObject())) .documentMapper(); MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); @@ -104,18 +116,18 @@ public void testTimestampNanos() throws IOException { Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper)timestampField).isEnabled()); + assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); } public void testBadTimestamp() throws IOException { - Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); String type = randomFrom("keyword", "integer", "long", "double", "text"); Exception e = expectThrows( IllegalArgumentException.class, - () -> createMapperService( - s, - mapping(b -> b.startObject("@timestamp").field("type", type).endObject()) - ) + () -> createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", type).endObject())) ); assertThat( e.getMessage(), @@ -123,6 +135,65 @@ public void testBadTimestamp() throws IOException { ); } + public void testEnabledTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject(DataStreamTimestampFieldMapper.NAME); + if (randomBoolean()) { + mappings.field("enabled", true); + } else { + mappings.field("enabled", "true"); + } + mappings.endObject().endObject().endObject(); + + DocumentMapper mapper = createMapperService(s, mappings).documentMapper(); + Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); + assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); + assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); + } + + public void testDisabledTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject(DataStreamTimestampFieldMapper.NAME) + .field("enabled", false) + .endObject() + .endObject() + .endObject(); + + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(s, mappings).documentMapper()); + assertThat( + e.getMessage(), + equalTo("Failed to parse mapping: time series index [_data_stream_timestamp] meta field must be enabled") + ); + } + + public void testBadTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .field(DataStreamTimestampFieldMapper.NAME, "enabled") + .endObject() + .endObject(); + + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(s, mappings).documentMapper()); + assertThat(e.getMessage(), equalTo("Failed to parse mapping: time series index [_data_stream_timestamp] meta field format error")); + } + public void testWithoutRoutingPath() { Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 35a80686b8635..0c713e008b1ab 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -294,7 +294,8 @@ public void testEmptyDocumentMapper() { Collections.sort(metadataMappers, Comparator.comparing(c -> c.getSimpleName())); assertMap( metadataMappers, - matchesList().item(DocCountFieldMapper.class) + matchesList().item(DataStreamTimestampFieldMapper.class) + .item(DocCountFieldMapper.class) .item(FieldNamesFieldMapper.class) .item(IdFieldMapper.class) .item(IgnoredFieldMapper.class) @@ -309,7 +310,8 @@ public void testEmptyDocumentMapper() { Collections.sort(matching); assertMap( matching, - matchesList().item(DocCountFieldMapper.CONTENT_TYPE) + matchesList().item(DataStreamTimestampFieldMapper.NAME) + .item(DocCountFieldMapper.CONTENT_TYPE) .item(FieldNamesFieldMapper.CONTENT_TYPE) .item(IdFieldMapper.CONTENT_TYPE) .item(IgnoredFieldMapper.CONTENT_TYPE) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 00575dcab32a9..08b366b8c3e4f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -177,7 +177,7 @@ public void testTotalFieldsLimitWithFieldAlias() throws Throwable { } public void testFieldNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(15, 20); + int maxFieldNameLength = randomIntBetween(25, 30); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); @@ -191,7 +191,7 @@ public void testFieldNameLengthLimit() throws Throwable { } public void testObjectNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(15, 20); + int maxFieldNameLength = randomIntBetween(25, 30); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); @@ -205,7 +205,7 @@ public void testObjectNameLengthLimit() throws Throwable { } public void testAliasFieldNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(15, 20); + int maxFieldNameLength = randomIntBetween(25, 30); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); @@ -221,7 +221,7 @@ public void testAliasFieldNameLengthLimit() throws Throwable { } public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(15, 20); + int maxFieldNameLength = randomIntBetween(25, 30); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index f9acded6340b5..a9c1329657cbf 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.indices; import org.elasticsearch.Version; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DocCountFieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; @@ -67,10 +68,18 @@ public Map getMetadataMappers() { } }); - private static final String[] EXPECTED_METADATA_FIELDS = new String[]{ IgnoredFieldMapper.NAME, IdFieldMapper.NAME, - RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, - NestedPathFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, DocCountFieldMapper.NAME, - FieldNamesFieldMapper.NAME }; + private static final String[] EXPECTED_METADATA_FIELDS = new String[] { + IgnoredFieldMapper.NAME, + IdFieldMapper.NAME, + RoutingFieldMapper.NAME, + IndexFieldMapper.NAME, + SourceFieldMapper.NAME, + NestedPathFieldMapper.NAME, + VersionFieldMapper.NAME, + SeqNoFieldMapper.NAME, + DocCountFieldMapper.NAME, + DataStreamTimestampFieldMapper.NAME, + FieldNamesFieldMapper.NAME }; public void testBuiltinMappers() { IndicesModule module = new IndicesModule(Collections.emptyList()); From 580bf51638ecd36f531cffcf41edd8fafdea4797 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 19 Oct 2021 10:15:46 +0800 Subject: [PATCH 4/7] fixed --- server/src/main/java/org/elasticsearch/index/IndexMode.java | 3 ++- .../mapper/MetadataCreateDataStreamServiceTests.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 1e46b21c60d70..4bc1014b924fd 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -115,7 +115,8 @@ public void completeMappings(MappingParserContext context, Map m DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, context.scriptCompiler(), DateFieldMapper.IGNORE_MALFORMED_SETTING.get(context.getSettings()), - context.getIndexSettings().getIndexVersionCreated() + context.getIndexSettings().getIndexVersionCreated(), + context.getIndexSettings().getIndex().getName() ) ); } diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java index f1d2a0bdaa4f5..ddccd3bed4bc1 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java @@ -74,7 +74,7 @@ MappingLookup createMappingLookup(String mapping) throws IOException { ) .putMapping(mapping) .build(); - IndicesModule indicesModule = new IndicesModule(List.of(new DataStreamsPlugin())); + IndicesModule indicesModule = new IndicesModule(List.of()); MapperService mapperService = MapperTestUtils.newMapperService( xContentRegistry(), createTempDir(), From 6d71f1f6a11cf3d04e20d0d89a903c0332f76f9a Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 19 Oct 2021 10:49:31 +0800 Subject: [PATCH 5/7] spotless --- .../datastreams/mapper/MetadataCreateDataStreamServiceTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java index ddccd3bed4bc1..dd0d350577ccd 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; import java.io.IOException; import java.util.List; From 2a6be9e45fb8bbe1bb70776e128addea12de17c3 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 19 Oct 2021 15:04:56 +0800 Subject: [PATCH 6/7] fixed --- .../template/ComposableTemplateIT.java | 27 ------------------- .../MetadataIndexTemplateService.java | 12 --------- 2 files changed, 39 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index b3d05e7b38f2e..8fdd61855c32c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -13,18 +13,10 @@ import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; -import java.io.IOException; import java.util.Collections; -import java.util.List; - -import static org.hamcrest.Matchers.equalTo; public class ComposableTemplateIT extends ESIntegTestCase { @@ -80,23 +72,4 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(cit2)).get(); } - - public void testUsageOfDataStreamFails() throws IOException { - // Exception that would happen if a unknown field is provided in a composable template: - // The thrown exception will be used to compare against the exception that is thrown when providing - // a composable template with a data stream definition. - String content = "{\"index_patterns\":[\"logs-*-*\"],\"my_field\":\"bla\"}"; - XContentParser parser = - XContentHelper.createParser(xContentRegistry(), null, new BytesArray(content), XContentType.JSON); - Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser)); - - ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of("logs-*-*")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()).build(); - Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, - new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet()); - Exception actualException = (Exception) e.getCause(); - assertThat(actualException.getMessage(), - equalTo(expectedException.getMessage().replace("[1:32] ", "").replace("my_field", "data_stream"))); - assertThat(actualException.getMessage(), equalTo("[index_template] unknown field [data_stream]")); - } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 56e41f89aa905..6b5bbf033a31b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -41,11 +41,9 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -1230,16 +1228,6 @@ private static void validateCompositeTemplate(final ClusterState state, String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed - if (template.getDataStreamTemplate() != null) { - // If there is no _data_stream meta field mapper and a data stream should be created then - // fail as if the data_stream field can't be parsed: - if (tempIndexService.mapperService().isMetadataField(DataStreamTimestampFieldMapper.NAME) == false) { - // Fail like a parsing expection, since we will be moving data_stream template out of server module and - // then we would fail with the same error message, like we do here. - throw new XContentParseException("[index_template] unknown field [data_stream]"); - } - } - List mappings = collectMappings(stateWithIndex, templateName, indexName, xContentRegistry); try { MapperService mapperService = tempIndexService.mapperService(); From 696a089d2ffcdfb23e92c8271d0370570aee777e Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 25 Oct 2021 23:59:52 +0800 Subject: [PATCH 7/7] spotless --- .../java/org/elasticsearch/index/TimeSeriesModeTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 6b258c92f339e..e48400182df95 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -18,13 +18,13 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperServiceTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.script.StringFieldScript.LeafFactory; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; import java.util.Map;