From 59694d5c481321efcdfa6fb888c025a0170a5d22 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 20 Sep 2021 11:35:54 -0400 Subject: [PATCH] TSDB: Automatically map timestamp If tsdb is enabled we need an `@timestamp` field. This automatically maps the field if it is missing and fails to create indices in time_series mode that map `@timestamp` as anything other than `date` and `date_nanos`. --- .../test/tsdb/15_timestamp_mapping.yml | 151 ++++++++++++++++++ .../org/elasticsearch/index/IndexMode.java | 34 ++++ .../index/mapper/ObjectMapper.java | 5 + .../index/mapper/RootObjectMapper.java | 1 + .../index/TimeSeriesModeTests.java | 50 +++++- 5 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml new file mode 100644 index 0000000000000..88002fc6c3228 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -0,0 +1,151 @@ + +--- +date: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + dimension: true + + - do: + indices.get_mapping: + index: test + - match: { "test.mappings.properties.@timestamp.type": date } + + - do: + bulk: + refresh: true + index: test_index + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}' + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: {hits.total.value: 1} + - match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] } + +--- +date_nanos: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date_nanos + metricset: + type: keyword + dimension: true + + - do: + indices.get_mapping: + index: test + - match: { "test.mappings.properties.@timestamp.type": date_nanos } + + - do: + bulk: + refresh: true + index: test_index + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}' + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: {hits.total.value: 1} + - match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] } + +--- +automatically add with date: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + metricset: + type: keyword + dimension: true + + - do: + indices.get_mapping: + index: test + - match: { 'test.mappings.properties.@timestamp': { "type": date } } + + - do: + bulk: + refresh: true + index: test_index + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}' + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: {hits.total.value: 1} + - match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] } + +--- +reject @timestamp with wrong type: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + catch: /@timestamp must be \[date\] or \[date_nanos\]/ + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: keyword diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index c92a5d742ec09..56fffb09cab04 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -11,10 +11,15 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MappingParserContext; +import org.elasticsearch.index.mapper.RootObjectMapper; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.stream.Stream; import static java.util.stream.Collectors.toSet; @@ -26,6 +31,9 @@ public enum IndexMode { STANDARD { @Override void validateWithOtherSettings(Map, Object> settings) {} + + @Override + public void completeMappings(MappingParserContext context, RootObjectMapper.Builder builder) {} }, TIME_SERIES { @Override @@ -43,6 +51,27 @@ void validateWithOtherSettings(Map, Object> settings) { private String error(Setting unsupported) { return "[" + IndexSettings.MODE.getKey() + "=time_series] is incompatible with [" + unsupported.getKey() + "]"; } + + @Override + public void completeMappings(MappingParserContext context, RootObjectMapper.Builder builder) { + Optional timestamp = builder.getBuilder("@timestamp"); + if (timestamp.isEmpty()) { + builder.add( + new DateFieldMapper.Builder( + "@timestamp", + DateFieldMapper.Resolution.MILLISECONDS, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + context.scriptCompiler(), + DateFieldMapper.IGNORE_MALFORMED_SETTING.get(context.getSettings()), + context.getIndexSettings().getIndexVersionCreated() + ) + ); + return; + } + if (false == timestamp.get() instanceof DateFieldMapper.Builder) { + throw new IllegalArgumentException("@timestamp must be [date] or [date_nanos]"); + } + } }; private static final List> TIME_SERIES_UNSUPPORTED = List.of( @@ -57,4 +86,9 @@ private String error(Setting unsupported) { ); abstract void validateWithOtherSettings(Map, Object> settings); + + /** + * Validate and/or modify the mappings after after they've been parsed. + */ + public abstract void completeMappings(MappingParserContext context, RootObjectMapper.Builder builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 6875ea191a4ff..cdf4d7639d40f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; public class ObjectMapper extends Mapper implements Cloneable { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ObjectMapper.class); @@ -86,6 +87,10 @@ public Builder add(Mapper.Builder builder) { return this; } + public Optional getBuilder(String name) { + return mappersBuilders.stream().filter(b -> b.name().equals(name)).findFirst(); + } + protected final Map buildMappers(boolean root, MapperBuilderContext context) { if (root == false) { context = context.createChildContext(name); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 2477e3fed5547..1e9a8c1c14db3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -151,6 +151,7 @@ public RootObjectMapper.Builder parse(String name, Map node, Map iterator.remove(); } } + parserContext.getIndexSettings().getMode().completeMappings(parserContext, builder); return builder; } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index c0e73bc833b42..5de7e8d1def9a 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -10,11 +10,19 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperServiceTestCase; + +import java.io.IOException; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; -public class TimeSeriesModeTests extends ESTestCase { +public class TimeSeriesModeTests extends MapperServiceTestCase { public void testPartitioned() { Settings s = Settings.builder() .put(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.getKey(), 2) @@ -50,4 +58,42 @@ public void testSortOrder() { Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s)); assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); } + + public void testAddsTimestamp() throws IOException { + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> {})).documentMapper(); + MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); + assertThat(timestamp, instanceOf(DateFieldType.class)); + assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); + } + + public void testTimestampMillis() throws IOException { + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) + .documentMapper(); + MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); + assertThat(timestamp, instanceOf(DateFieldType.class)); + assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); + } + + public void testTimestampNanos() throws IOException { + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date_nanos").endObject())) + .documentMapper(); + MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); + assertThat(timestamp, instanceOf(DateFieldType.class)); + assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.NANOSECONDS)); + } + + public void testBadTimestamp() throws IOException { + Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); + Exception e = expectThrows( + MapperParsingException.class, + () -> createMapperService( + s, + mapping(b -> b.startObject("@timestamp").field("type", randomFrom("keyword", "int", "long", "double", "text")).endObject()) + ) + ); + assertThat(e.getMessage(), equalTo("Failed to parse mapping: @timestamp must be [date] or [date_nanos]")); + } }