From 2bad3899c3b1a9d0b2e67d6b39d24d031b4f6484 Mon Sep 17 00:00:00 2001 From: weizijun Date: Fri, 24 Sep 2021 15:09:51 +0800 Subject: [PATCH 01/24] add settings --- .../common/settings/IndexScopedSettings.java | 2 + .../elasticsearch/index/IndexSettings.java | 51 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 75cd526920222..2a0e7323321dd 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -185,6 +185,8 @@ private static Set> builtInIndexSettings() { } Set> result = new HashSet<>(ALWAYS_ENABLED_BUILT_IN_INDEX_SETTINGS); result.add(IndexSettings.MODE); + result.add(IndexSettings.TIME_SERIES_START_TIME); + result.add(IndexSettings.TIME_SERIES_END_TIME); return Set.copyOf(result); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index bd8f180368c7f..cd942e8ba8c32 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -363,6 +363,20 @@ public Iterator> settings() { Property.Final ); + public static final Setting TIME_SERIES_START_TIME = Setting.timeSetting( + "index.time_series.start_time", + new TimeValue(-1, TimeUnit.MILLISECONDS), + Property.IndexScope, + Property.Dynamic + ); + + public static final Setting TIME_SERIES_END_TIME = Setting.timeSetting( + "index.time_series.end_time", + new TimeValue(-1, TimeUnit.MILLISECONDS), + Property.IndexScope, + Property.Dynamic + ); + private final Index index; private final Version version; private final Logger logger; @@ -373,6 +387,8 @@ public Iterator> settings() { * The {@link IndexMode "mode"} of the index. */ private final IndexMode mode; + private volatile TimeValue timeSeriesStartTime; + private volatile TimeValue timeSeriesEndTime; // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock private volatile Settings settings; @@ -515,6 +531,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD; + timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings); + timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings); this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings); this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings); @@ -614,6 +632,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit); + scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_START_TIME, this::updateTimeSeriesStartTime); + scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_END_TIME, this::updateTimeSeriesEndTime); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { this.searchIdleAfter = searchIdleAfter; } @@ -1092,4 +1112,35 @@ public long getMappingDimensionFieldsLimit() { private void setMappingDimensionFieldsLimit(long value) { this.mappingDimensionFieldsLimit = value; } + + public TimeValue getTimeSeriesStartTime() { + return timeSeriesStartTime; + } + + public void updateTimeSeriesStartTime(TimeValue startTime) { + if (true == this.timeSeriesStartTime.equals(TimeValue.MINUS_ONE)) { + throw new IllegalArgumentException("index.time_series.start_time not set before, can not update value"); + } + + if (this.timeSeriesStartTime.getMicros() < startTime.getMicros()) { + throw new IllegalArgumentException("index.time_series.start_time must smaller then pre value [" + this.timeSeriesStartTime + "]"); + } + this.timeSeriesStartTime = startTime; + } + + public TimeValue getTimeSeriesEndTime() { + return timeSeriesEndTime; + } + + public void updateTimeSeriesEndTime(TimeValue endTime) { + if (true == this.timeSeriesEndTime.equals(TimeValue.MINUS_ONE)) { + throw new IllegalArgumentException("index.time_series.end_time not set before, can not update value"); + } + + if (this.timeSeriesEndTime.getMicros() > endTime.getMicros()) { + throw new IllegalArgumentException("index.time_series.end_time must large then pre value [" + this.timeSeriesEndTime + "]"); + } + + this.timeSeriesEndTime = endTime; + } } From 88097573c2e3ed56e1b9890073238cc5b554d54e Mon Sep 17 00:00:00 2001 From: weizijun Date: Sat, 25 Sep 2021 00:39:38 +0800 Subject: [PATCH 02/24] add check code --- .../org/elasticsearch/index/IndexMode.java | 52 ++++++++++ .../elasticsearch/index/IndexSettings.java | 36 ++++--- .../index/mapper/DocumentParser.java | 2 + .../index/IndexSettingsTests.java | 64 ++++++++++++ .../index/TimeSeriesModeTests.java | 98 ++++++++++++++++++- 5 files changed, 234 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index c92a5d742ec09..e7866723b90c8 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -8,16 +8,22 @@ package org.elasticsearch.index; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DocumentParserContext; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Stream; import static java.util.stream.Collectors.toSet; +import static org.elasticsearch.core.TimeValue.NSEC_PER_MSEC; /** * "Mode" that controls which behaviors and settings an index supports. @@ -26,8 +32,13 @@ public enum IndexMode { STANDARD { @Override void validateWithOtherSettings(Map, Object> settings) {} + + @Override + public void validateWithSource(DocumentParserContext context) {} }, TIME_SERIES { + public static final String TIMESTAMP_FIELD = "@timestamp"; + @Override void validateWithOtherSettings(Map, Object> settings) { if (settings.get(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING) != Integer.valueOf(1)) { @@ -40,6 +51,45 @@ void validateWithOtherSettings(Map, Object> settings) { } } + @Override + public void validateWithSource(DocumentParserContext context) { + validateTimestamp(context); + } + + private void validateTimestamp(DocumentParserContext context) { + IndexableField[] fields = context.rootDoc().getFields(TIMESTAMP_FIELD); + if (fields.length == 0) { + throw new IllegalArgumentException("time series index @timestamp field is missing"); + } + + long numberOfValues = Arrays.stream(fields) + .filter(indexableField -> indexableField.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) + .count(); + if (numberOfValues > 1) { + throw new IllegalArgumentException("time series index @timestamp field encountered multiple values"); + } + + long timestamp = fields[0].numericValue().longValue(); + if (context.mappingLookup().getMapper(TIMESTAMP_FIELD).typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) { + timestamp /= NSEC_PER_MSEC; + } + + long startTime = context.indexSettings().getTimeSeriesStartTime(); + long endTime = context.indexSettings().getTimeSeriesEndTime(); + + if (timestamp < startTime) { + throw new IllegalArgumentException( + "time series index @timestamp value [" + timestamp + "] must be larger than " + startTime + ); + } + + if (timestamp > endTime) { + throw new IllegalArgumentException( + "time series index @timestamp value [" + timestamp + "] must be smaller than " + endTime + ); + } + } + private String error(Setting unsupported) { return "[" + IndexSettings.MODE.getKey() + "=time_series] is incompatible with [" + unsupported.getKey() + "]"; } @@ -57,4 +107,6 @@ private String error(Setting unsupported) { ); abstract void validateWithOtherSettings(Map, Object> settings); + + public abstract void validateWithSource(DocumentParserContext context); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index cd942e8ba8c32..c10b4ecd547bf 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -363,16 +363,18 @@ public Iterator> settings() { Property.Final ); - public static final Setting TIME_SERIES_START_TIME = Setting.timeSetting( + public static final Setting TIME_SERIES_START_TIME = Setting.longSetting( "index.time_series.start_time", - new TimeValue(-1, TimeUnit.MILLISECONDS), + -1L, + -1L, Property.IndexScope, Property.Dynamic ); - public static final Setting TIME_SERIES_END_TIME = Setting.timeSetting( + public static final Setting TIME_SERIES_END_TIME = Setting.longSetting( "index.time_series.end_time", - new TimeValue(-1, TimeUnit.MILLISECONDS), + -1L, + -1L, Property.IndexScope, Property.Dynamic ); @@ -387,8 +389,8 @@ public Iterator> settings() { * The {@link IndexMode "mode"} of the index. */ private final IndexMode mode; - private volatile TimeValue timeSeriesStartTime; - private volatile TimeValue timeSeriesEndTime; + private volatile long timeSeriesStartTime; + private volatile long timeSeriesEndTime; // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock private volatile Settings settings; @@ -1113,32 +1115,34 @@ private void setMappingDimensionFieldsLimit(long value) { this.mappingDimensionFieldsLimit = value; } - public TimeValue getTimeSeriesStartTime() { + public long getTimeSeriesStartTime() { return timeSeriesStartTime; } - public void updateTimeSeriesStartTime(TimeValue startTime) { - if (true == this.timeSeriesStartTime.equals(TimeValue.MINUS_ONE)) { + public void updateTimeSeriesStartTime(long startTime) { + if (timeSeriesStartTime < 0) { throw new IllegalArgumentException("index.time_series.start_time not set before, can not update value"); } - if (this.timeSeriesStartTime.getMicros() < startTime.getMicros()) { - throw new IllegalArgumentException("index.time_series.start_time must smaller then pre value [" + this.timeSeriesStartTime + "]"); + if (this.timeSeriesStartTime < startTime) { + throw new IllegalArgumentException( + "index.time_series.start_time must be smaller than pre value [" + this.timeSeriesStartTime + "]" + ); } this.timeSeriesStartTime = startTime; } - public TimeValue getTimeSeriesEndTime() { + public long getTimeSeriesEndTime() { return timeSeriesEndTime; } - public void updateTimeSeriesEndTime(TimeValue endTime) { - if (true == this.timeSeriesEndTime.equals(TimeValue.MINUS_ONE)) { + public void updateTimeSeriesEndTime(long endTime) { + if (timeSeriesEndTime < 0) { throw new IllegalArgumentException("index.time_series.end_time not set before, can not update value"); } - if (this.timeSeriesEndTime.getMicros() > endTime.getMicros()) { - throw new IllegalArgumentException("index.time_series.end_time must large then pre value [" + this.timeSeriesEndTime + "]"); + if (this.timeSeriesEndTime > endTime) { + throw new IllegalArgumentException("index.time_series.end_time must be larger than pre value [" + this.timeSeriesEndTime + "]"); } this.timeSeriesEndTime = endTime; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 155d6e1b33d72..40390007fbf9b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -139,6 +139,8 @@ private static void internalParseDocument(RootObjectMapper root, MetadataFieldMa executeIndexTimeScripts(context); + context.indexSettings().getMode().validateWithSource(context); + for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) { metadataMapper.postParse(context); } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 6d596e15ed9fb..15c2985b99420 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -15,11 +15,13 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; +import org.hamcrest.Matchers; import java.util.Arrays; import java.util.Collections; @@ -545,4 +547,66 @@ public void testCustomDataPathDeprecated() { assertThat(indexSettings.hasCustomDataPath(), is(true)); assertSettingDeprecationsAndWarnings(new Setting[] { IndexMetadata.INDEX_DATA_PATH_SETTING }); } + + public void testUpdateTimeSeriesTimeRange() { + long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000); + long endTime = startTime + randomLongBetween(1000, 86400000); + final Settings settings = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .build(); + IndexMetadata metadata = newIndexMeta("test", settings); + IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); + + // test update start_time + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesStartTime(startTime + randomLongBetween(10, 1000)) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time must be smaller than pre value")); + + // success + long newStartTime = Math.max(startTime - randomLongBetween(10, 1000), 0); + indexSettings.updateTimeSeriesStartTime(newStartTime); + assertEquals(newStartTime, indexSettings.getTimeSeriesStartTime()); + } + + // test update end_time + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesEndTime(endTime - randomLongBetween(10, 1000)) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than pre value")); + + // success + long newEndTime = endTime + randomLongBetween(10, 1000); + indexSettings.updateTimeSeriesEndTime(newEndTime); + assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); + } + } + + public void testUpdateTimeSeriesTimeRangeNotSet() { + final Settings settings = Settings.builder().build(); + IndexMetadata metadata = newIndexMeta("test", settings); + IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); + + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesStartTime(randomLongBetween(10, 1000)) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time not set before, can not update value")); + } + + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesEndTime(randomLongBetween(10, 1000)) + ); + + assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time not set before, can not update value")); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index c0e73bc833b42..1e360714cd03e 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -9,12 +9,22 @@ package org.elasticsearch.index; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.common.time.DateUtils; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.SourceToParse; +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -public class TimeSeriesModeTests extends ESTestCase { +public class TimeSeriesModeTests extends MapperServiceTestCase { public void testPartitioned() { Settings s = Settings.builder() .put(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.getKey(), 2) @@ -50,4 +60,88 @@ public void testSortOrder() { Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s)); assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); } + + public void testValidateTimestamp() throws IOException { + long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000); + long endTime = startTime + randomLongBetween(1000, 86400000); + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .build(); + + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) + .documentMapper(); + // timestamp missing + { + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()), + XContentType.JSON + ) + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("time series index @timestamp field is missing")); + } + + // timestamp smaller than start_time + { + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .field("@timestamp", Math.max(startTime - randomLongBetween(10, 1000), 0)) + .endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("must be larger than")); + } + + // timestamp larger than end_time + { + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .field("@timestamp", endTime + randomLongBetween(10, 1000)) + .endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("must be smaller than")); + } + + // timestamp correct + { + mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("@timestamp", randomLongBetween(startTime, endTime)).endObject() + ), + XContentType.JSON + ) + ); + } + } } From 6c42357b815944c9cc81757258faf1536e66c9bf Mon Sep 17 00:00:00 2001 From: weizijun Date: Sat, 25 Sep 2021 00:51:50 +0800 Subject: [PATCH 03/24] add some comment --- .../main/java/org/elasticsearch/index/IndexSettings.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index c10b4ecd547bf..135b16feac50f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -363,6 +363,9 @@ public Iterator> settings() { Property.Final ); + /** + * in time series mode, the start time of the index, timestamp must larger than start_time + */ public static final Setting TIME_SERIES_START_TIME = Setting.longSetting( "index.time_series.start_time", -1L, @@ -371,6 +374,9 @@ public Iterator> settings() { Property.Dynamic ); + /** + * in time series mode, the end time of the index, timestamp must smaller than start_time + */ public static final Setting TIME_SERIES_END_TIME = Setting.longSetting( "index.time_series.end_time", -1L, From c3f1e66e28264c6c4e7d1c51a540f586d55dc682 Mon Sep 17 00:00:00 2001 From: weizijun Date: Sun, 26 Sep 2021 12:03:41 +0800 Subject: [PATCH 04/24] add yaml tests --- .../rest-api-spec/test/tsdb/10_settings.yml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index d9dd72dfc4a2f..ea68831c09250 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -117,3 +117,50 @@ no partitioning: mode: time_series shards: 5 routing_partition_size: 2 + +--- +set start_time and end_time: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + - do: + indices.create: + index: test_index + body: + settings: + index: + mode: time_series + time_series: + start_time: 1632625782000 + end_time: 1632625792000 + + - do: + indices.put_settings: + index: test_index + body: + index: + time_series: + start_time: 1632625781000 + end_time: 1632625793000 + + - do: + catch: /index.time_series.start_time must be smaller than pre value \[1632625781000\]/ + indices.put_settings: + index: test_index + body: + index: + time_series: + start_time: 1632625783000 + + - do: + catch: /index.time_series.end_time must be larger than pre value \[1632625793000\]/ + indices.put_settings: + index: test_index + body: + index: + time_series: + end_time: 1632625792000 + + - do: + indices.delete: + index: test_index From 79e96e097d0bd4b65a7d1903c5f11962d0f9b106 Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 27 Sep 2021 14:48:22 +0800 Subject: [PATCH 05/24] add the time_series mode check --- .../rest-api-spec/test/tsdb/10_settings.yml | 25 +++++++++++ .../common/settings/Setting.java | 6 +++ .../elasticsearch/index/IndexSettings.java | 44 ++++++++++++++++++- .../index/IndexSettingsTests.java | 36 +++++++++++++-- 4 files changed, 106 insertions(+), 5 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index ea68831c09250..7ccc8aba525bd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -164,3 +164,28 @@ set start_time and end_time: - do: indices.delete: index: test_index + +--- +set start_time and end_time without timeseries mode: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + - do: + catch: /index.time_series.start_time need to be used for time_series mode/ + indices.create: + index: test_index + body: + settings: + index: + time_series: + start_time: 1632625782000 + + - do: + catch: /index.time_series.end_time need to be used for time_series mode/ + indices.create: + index: test_index + body: + settings: + index: + time_series: + end_time: 1632625782000 diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 19623018d048b..5eb5a5f91ee98 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1222,6 +1222,12 @@ public static Setting longSetting(String key, long defaultValue, long minV properties); } + public static Setting longSetting(String key, long defaultValue, long minValue, Validator validator, + Property... properties) { + return new Setting<>(key, Long.toString(defaultValue), (s) -> parseLong(s, minValue, key, isFiltered(properties)), validator, + properties); + } + public static Setting simpleString(String key, Property... properties) { return new Setting<>(key, s -> "", Function.identity(), properties); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 135b16feac50f..36559405d4c90 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -365,22 +365,60 @@ public Iterator> settings() { /** * in time series mode, the start time of the index, timestamp must larger than start_time + * format is epoch_millis */ public static final Setting TIME_SERIES_START_TIME = Setting.longSetting( "index.time_series.start_time", -1L, -1L, + new Setting.Validator<>() { + @Override + public void validate(Long value) {} + + @Override + public void validate(Long value, Map, Object> settings) { + IndexMode mode = (IndexMode) settings.get(MODE); + if (mode != IndexMode.TIME_SERIES) { + throw new IllegalArgumentException("index.time_series.start_time need to be used for time_series mode"); + } + } + + @Override + public Iterator> settings() { + final List> settings = List.of(MODE); + return settings.iterator(); + } + }, Property.IndexScope, Property.Dynamic ); /** * in time series mode, the end time of the index, timestamp must smaller than start_time + * format is epoch_millis */ public static final Setting TIME_SERIES_END_TIME = Setting.longSetting( "index.time_series.end_time", -1L, -1L, + new Setting.Validator<>() { + @Override + public void validate(Long value) {} + + @Override + public void validate(Long value, Map, Object> settings) { + IndexMode mode = (IndexMode) settings.get(MODE); + if (mode != IndexMode.TIME_SERIES) { + throw new IllegalArgumentException("index.time_series.end_time need to be used for time_series mode"); + } + } + + @Override + public Iterator> settings() { + final List> settings = List.of(MODE); + return settings.iterator(); + } + }, Property.IndexScope, Property.Dynamic ); @@ -539,8 +577,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD; - timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings); - timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings); + if (mode == IndexMode.TIME_SERIES) { + timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings); + timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings); + } this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings); this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 15c2985b99420..5c70c3f904ec7 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -31,6 +31,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import static org.elasticsearch.index.IndexSettings.TIME_SERIES_END_TIME; +import static org.elasticsearch.index.IndexSettings.TIME_SERIES_START_TIME; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.StringContains.containsString; @@ -552,8 +554,9 @@ public void testUpdateTimeSeriesTimeRange() { long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000); long endTime = startTime + randomLongBetween(1000, 86400000); final Settings settings = Settings.builder() - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(TIME_SERIES_START_TIME.getKey(), startTime) + .put(TIME_SERIES_END_TIME.getKey(), endTime) .build(); IndexMetadata metadata = newIndexMeta("test", settings); IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); @@ -588,7 +591,9 @@ public void testUpdateTimeSeriesTimeRange() { } public void testUpdateTimeSeriesTimeRangeNotSet() { - final Settings settings = Settings.builder().build(); + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .build(); IndexMetadata metadata = newIndexMeta("test", settings); IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); @@ -609,4 +614,29 @@ public void testUpdateTimeSeriesTimeRangeNotSet() { assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time not set before, can not update value")); } } + + public void testTimeSeriesTimeRangeModeSet() { + long time = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999); + { + final Settings settings = Settings.builder() + .put(TIME_SERIES_START_TIME.getKey(), time) + .build(); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> TIME_SERIES_START_TIME.get(settings) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time need to be used for time_series mode")); + } + + { + final Settings settings = Settings.builder() + .put(TIME_SERIES_END_TIME.getKey(), time) + .build(); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> TIME_SERIES_END_TIME.get(settings) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time need to be used for time_series mode")); + } + } } From b4a68c3650d6c6a1211952237d0a894b1c4209dd Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 28 Sep 2021 19:15:46 +0800 Subject: [PATCH 06/24] improve --- .../rest-api-spec/test/tsdb/10_settings.yml | 12 +-- .../common/settings/Setting.java | 22 +++-- .../org/elasticsearch/index/IndexMode.java | 2 +- .../elasticsearch/index/IndexSettings.java | 69 ++++++-------- .../index/IndexSettingsTests.java | 90 ++++++------------- .../index/TimeSeriesModeTests.java | 11 +-- 6 files changed, 76 insertions(+), 130 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index 7ccc8aba525bd..fb6c4e14028c3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -140,20 +140,10 @@ set start_time and end_time: body: index: time_series: - start_time: 1632625781000 end_time: 1632625793000 - do: - catch: /index.time_series.start_time must be smaller than pre value \[1632625781000\]/ - indices.put_settings: - index: test_index - body: - index: - time_series: - start_time: 1632625783000 - - - do: - catch: /index.time_series.end_time must be larger than pre value \[1632625793000\]/ + catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/ indices.put_settings: index: test_index body: diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 5eb5a5f91ee98..be727b632ba0f 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -12,15 +12,11 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.MemorySizeValue; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -29,8 +25,14 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.mapper.DateFieldMapper; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -1222,10 +1224,14 @@ public static Setting longSetting(String key, long defaultValue, long minV properties); } - public static Setting longSetting(String key, long defaultValue, long minValue, Validator validator, - Property... properties) { - return new Setting<>(key, Long.toString(defaultValue), (s) -> parseLong(s, minValue, key, isFiltered(properties)), validator, - properties); + public static Setting dateSetting(String key, Instant defaultValue, Validator validator, Property... properties) { + return new Setting<>( + key, + defaultValue.toString(), + (s) -> Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(s)), + validator, + properties + ); } public static Setting simpleString(String key, Property... properties) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index e7866723b90c8..60356add0bb2b 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -83,7 +83,7 @@ private void validateTimestamp(DocumentParserContext context) { ); } - if (timestamp > endTime) { + if (timestamp >= endTime) { throw new IllegalArgumentException( "time series index @timestamp value [" + timestamp + "] must be smaller than " + endTime ); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 36559405d4c90..a08861cfe52ba 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Booleans; @@ -26,6 +27,7 @@ import org.elasticsearch.ingest.IngestService; import org.elasticsearch.node.Node; +import java.time.Instant; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -365,18 +367,16 @@ public Iterator> settings() { /** * in time series mode, the start time of the index, timestamp must larger than start_time - * format is epoch_millis */ - public static final Setting TIME_SERIES_START_TIME = Setting.longSetting( + public static final Setting TIME_SERIES_START_TIME = Setting.dateSetting( "index.time_series.start_time", - -1L, - -1L, + Instant.ofEpochMilli(0), new Setting.Validator<>() { @Override - public void validate(Long value) {} + public void validate(Instant value) {} @Override - public void validate(Long value, Map, Object> settings) { + public void validate(Instant value, Map, Object> settings) { IndexMode mode = (IndexMode) settings.get(MODE); if (mode != IndexMode.TIME_SERIES) { throw new IllegalArgumentException("index.time_series.start_time need to be used for time_series mode"); @@ -389,33 +389,35 @@ public Iterator> settings() { return settings.iterator(); } }, - Property.IndexScope, - Property.Dynamic + Property.IndexScope ); /** * in time series mode, the end time of the index, timestamp must smaller than start_time - * format is epoch_millis */ - public static final Setting TIME_SERIES_END_TIME = Setting.longSetting( + public static final Setting TIME_SERIES_END_TIME = Setting.dateSetting( "index.time_series.end_time", - -1L, - -1L, + DateUtils.MAX_NANOSECOND_INSTANT, new Setting.Validator<>() { @Override - public void validate(Long value) {} + public void validate(Instant value) {} @Override - public void validate(Long value, Map, Object> settings) { + public void validate(Instant value, Map, Object> settings) { IndexMode mode = (IndexMode) settings.get(MODE); if (mode != IndexMode.TIME_SERIES) { throw new IllegalArgumentException("index.time_series.end_time need to be used for time_series mode"); } + + Instant startTime = (Instant) settings.get(TIME_SERIES_START_TIME); + if (startTime.toEpochMilli() > value.toEpochMilli()) { + throw new IllegalArgumentException("index.time_series.end_time must be larger than index.time_series.start_time"); + } } @Override public Iterator> settings() { - final List> settings = List.of(MODE); + final List> settings = List.of(MODE, TIME_SERIES_START_TIME); return settings.iterator(); } }, @@ -433,7 +435,7 @@ public Iterator> settings() { * The {@link IndexMode "mode"} of the index. */ private final IndexMode mode; - private volatile long timeSeriesStartTime; + private final long timeSeriesStartTime; private volatile long timeSeriesEndTime; // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock @@ -577,10 +579,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD; - if (mode == IndexMode.TIME_SERIES) { - timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings); - timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings); - } + timeSeriesStartTime = mode == IndexMode.TIME_SERIES ? TIME_SERIES_START_TIME.get(settings).toEpochMilli() : -1L; + timeSeriesEndTime = mode == IndexMode.TIME_SERIES ? TIME_SERIES_END_TIME.get(settings).toEpochMilli() : -1L; this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings); this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings); @@ -680,7 +680,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit); - scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_START_TIME, this::updateTimeSeriesStartTime); scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_END_TIME, this::updateTimeSeriesEndTime); } @@ -1165,32 +1164,18 @@ public long getTimeSeriesStartTime() { return timeSeriesStartTime; } - public void updateTimeSeriesStartTime(long startTime) { - if (timeSeriesStartTime < 0) { - throw new IllegalArgumentException("index.time_series.start_time not set before, can not update value"); - } - - if (this.timeSeriesStartTime < startTime) { - throw new IllegalArgumentException( - "index.time_series.start_time must be smaller than pre value [" + this.timeSeriesStartTime + "]" - ); - } - this.timeSeriesStartTime = startTime; - } - public long getTimeSeriesEndTime() { return timeSeriesEndTime; } - public void updateTimeSeriesEndTime(long endTime) { - if (timeSeriesEndTime < 0) { - throw new IllegalArgumentException("index.time_series.end_time not set before, can not update value"); - } - - if (this.timeSeriesEndTime > endTime) { - throw new IllegalArgumentException("index.time_series.end_time must be larger than pre value [" + this.timeSeriesEndTime + "]"); + public void updateTimeSeriesEndTime(Instant endTime) { + long updateEndTime = endTime.toEpochMilli(); + if (this.timeSeriesEndTime > updateEndTime) { + throw new IllegalArgumentException( + "index.time_series.end_time must be larger than current value [" + this.timeSeriesEndTime + "]" + ); } - this.timeSeriesEndTime = endTime; + this.timeSeriesEndTime = updateEndTime; } } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 5c70c3f904ec7..c6fbbbdbe4b2c 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.translog.Translog; @@ -23,6 +22,7 @@ import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -551,8 +551,8 @@ public void testCustomDataPathDeprecated() { } public void testUpdateTimeSeriesTimeRange() { - long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000); - long endTime = startTime + randomLongBetween(1000, 86400000); + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put(TIME_SERIES_START_TIME.getKey(), startTime) @@ -561,81 +561,45 @@ public void testUpdateTimeSeriesTimeRange() { IndexMetadata metadata = newIndexMeta("test", settings); IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - // test update start_time - { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesStartTime(startTime + randomLongBetween(10, 1000)) - ); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time must be smaller than pre value")); - - // success - long newStartTime = Math.max(startTime - randomLongBetween(10, 1000), 0); - indexSettings.updateTimeSeriesStartTime(newStartTime); - assertEquals(newStartTime, indexSettings.getTimeSeriesStartTime()); - } - // test update end_time - { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesEndTime(endTime - randomLongBetween(10, 1000)) - ); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than pre value")); - - // success - long newEndTime = endTime + randomLongBetween(10, 1000); - indexSettings.updateTimeSeriesEndTime(newEndTime); - assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); - } + // smaller + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(endTime - randomLongBetween(1, 1000))) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than current value")); + + // success + long newEndTime = endTime + randomLongBetween(1, 1000); + indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(newEndTime)); + assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); } - public void testUpdateTimeSeriesTimeRangeNotSet() { + public void testTimeSeriesTimeBoundary() { + long startTime = System.currentTimeMillis(); + long endTime = startTime - randomLongBetween(1, 1000); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(TIME_SERIES_START_TIME.getKey(), startTime) + .put(TIME_SERIES_END_TIME.getKey(), endTime) .build(); IndexMetadata metadata = newIndexMeta("test", settings); - IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesStartTime(randomLongBetween(10, 1000)) - ); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time not set before, can not update value")); - } - - { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesEndTime(randomLongBetween(10, 1000)) - ); - - assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time not set before, can not update value")); - } + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than index.time_series.start_time")); } - public void testTimeSeriesTimeRangeModeSet() { - long time = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999); + public void testTimeSeriesTimeRangeModeNotSet() { + long time = System.currentTimeMillis(); { - final Settings settings = Settings.builder() - .put(TIME_SERIES_START_TIME.getKey(), time) - .build(); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> TIME_SERIES_START_TIME.get(settings) - ); + final Settings settings = Settings.builder().put(TIME_SERIES_START_TIME.getKey(), time).build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TIME_SERIES_START_TIME.get(settings)); assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time need to be used for time_series mode")); } { - final Settings settings = Settings.builder() - .put(TIME_SERIES_END_TIME.getKey(), time) - .build(); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> TIME_SERIES_END_TIME.get(settings) - ); + final Settings settings = Settings.builder().put(TIME_SERIES_END_TIME.getKey(), time).build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TIME_SERIES_END_TIME.get(settings)); assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time need to be used for time_series mode")); } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 1e360714cd03e..7ab5906c10a2c 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; @@ -20,6 +19,7 @@ import org.elasticsearch.index.mapper.SourceToParse; import java.io.IOException; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -62,8 +62,9 @@ public void testSortOrder() { } public void testValidateTimestamp() throws IOException { - long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000); - long endTime = startTime + randomLongBetween(1000, 86400000); + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + Settings s = Settings.builder() .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) @@ -99,7 +100,7 @@ public void testValidateTimestamp() throws IOException { BytesReference.bytes( XContentFactory.jsonBuilder() .startObject() - .field("@timestamp", Math.max(startTime - randomLongBetween(10, 1000), 0)) + .field("@timestamp", Math.max(startTime - randomLongBetween(1, 10), 0)) .endObject() ), XContentType.JSON @@ -120,7 +121,7 @@ public void testValidateTimestamp() throws IOException { BytesReference.bytes( XContentFactory.jsonBuilder() .startObject() - .field("@timestamp", endTime + randomLongBetween(10, 1000)) + .field("@timestamp", endTime + randomLongBetween(0, 10)) .endObject() ), XContentType.JSON From 63e552cbca89f94c41d7e603379fdbe90041eabe Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 28 Sep 2021 19:18:16 +0800 Subject: [PATCH 07/24] Revert "improve" This reverts commit b4a68c3650d6c6a1211952237d0a894b1c4209dd. --- .../rest-api-spec/test/tsdb/10_settings.yml | 12 ++- .../common/settings/Setting.java | 22 ++--- .../org/elasticsearch/index/IndexMode.java | 2 +- .../elasticsearch/index/IndexSettings.java | 69 ++++++++------ .../index/IndexSettingsTests.java | 90 +++++++++++++------ .../index/TimeSeriesModeTests.java | 11 ++- 6 files changed, 130 insertions(+), 76 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index fb6c4e14028c3..7ccc8aba525bd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -140,10 +140,20 @@ set start_time and end_time: body: index: time_series: + start_time: 1632625781000 end_time: 1632625793000 - do: - catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/ + catch: /index.time_series.start_time must be smaller than pre value \[1632625781000\]/ + indices.put_settings: + index: test_index + body: + index: + time_series: + start_time: 1632625783000 + + - do: + catch: /index.time_series.end_time must be larger than pre value \[1632625793000\]/ indices.put_settings: index: test_index body: diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index be727b632ba0f..5eb5a5f91ee98 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -12,11 +12,15 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.MemorySizeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -25,14 +29,8 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.mapper.DateFieldMapper; import java.io.IOException; -import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -1224,14 +1222,10 @@ public static Setting longSetting(String key, long defaultValue, long minV properties); } - public static Setting dateSetting(String key, Instant defaultValue, Validator validator, Property... properties) { - return new Setting<>( - key, - defaultValue.toString(), - (s) -> Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(s)), - validator, - properties - ); + public static Setting longSetting(String key, long defaultValue, long minValue, Validator validator, + Property... properties) { + return new Setting<>(key, Long.toString(defaultValue), (s) -> parseLong(s, minValue, key, isFiltered(properties)), validator, + properties); } public static Setting simpleString(String key, Property... properties) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 60356add0bb2b..e7866723b90c8 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -83,7 +83,7 @@ private void validateTimestamp(DocumentParserContext context) { ); } - if (timestamp >= endTime) { + if (timestamp > endTime) { throw new IllegalArgumentException( "time series index @timestamp value [" + timestamp + "] must be smaller than " + endTime ); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index a08861cfe52ba..36559405d4c90 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Booleans; @@ -27,7 +26,6 @@ import org.elasticsearch.ingest.IngestService; import org.elasticsearch.node.Node; -import java.time.Instant; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -367,16 +365,18 @@ public Iterator> settings() { /** * in time series mode, the start time of the index, timestamp must larger than start_time + * format is epoch_millis */ - public static final Setting TIME_SERIES_START_TIME = Setting.dateSetting( + public static final Setting TIME_SERIES_START_TIME = Setting.longSetting( "index.time_series.start_time", - Instant.ofEpochMilli(0), + -1L, + -1L, new Setting.Validator<>() { @Override - public void validate(Instant value) {} + public void validate(Long value) {} @Override - public void validate(Instant value, Map, Object> settings) { + public void validate(Long value, Map, Object> settings) { IndexMode mode = (IndexMode) settings.get(MODE); if (mode != IndexMode.TIME_SERIES) { throw new IllegalArgumentException("index.time_series.start_time need to be used for time_series mode"); @@ -389,35 +389,33 @@ public Iterator> settings() { return settings.iterator(); } }, - Property.IndexScope + Property.IndexScope, + Property.Dynamic ); /** * in time series mode, the end time of the index, timestamp must smaller than start_time + * format is epoch_millis */ - public static final Setting TIME_SERIES_END_TIME = Setting.dateSetting( + public static final Setting TIME_SERIES_END_TIME = Setting.longSetting( "index.time_series.end_time", - DateUtils.MAX_NANOSECOND_INSTANT, + -1L, + -1L, new Setting.Validator<>() { @Override - public void validate(Instant value) {} + public void validate(Long value) {} @Override - public void validate(Instant value, Map, Object> settings) { + public void validate(Long value, Map, Object> settings) { IndexMode mode = (IndexMode) settings.get(MODE); if (mode != IndexMode.TIME_SERIES) { throw new IllegalArgumentException("index.time_series.end_time need to be used for time_series mode"); } - - Instant startTime = (Instant) settings.get(TIME_SERIES_START_TIME); - if (startTime.toEpochMilli() > value.toEpochMilli()) { - throw new IllegalArgumentException("index.time_series.end_time must be larger than index.time_series.start_time"); - } } @Override public Iterator> settings() { - final List> settings = List.of(MODE, TIME_SERIES_START_TIME); + final List> settings = List.of(MODE); return settings.iterator(); } }, @@ -435,7 +433,7 @@ public Iterator> settings() { * The {@link IndexMode "mode"} of the index. */ private final IndexMode mode; - private final long timeSeriesStartTime; + private volatile long timeSeriesStartTime; private volatile long timeSeriesEndTime; // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock @@ -579,8 +577,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD; - timeSeriesStartTime = mode == IndexMode.TIME_SERIES ? TIME_SERIES_START_TIME.get(settings).toEpochMilli() : -1L; - timeSeriesEndTime = mode == IndexMode.TIME_SERIES ? TIME_SERIES_END_TIME.get(settings).toEpochMilli() : -1L; + if (mode == IndexMode.TIME_SERIES) { + timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings); + timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings); + } this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings); this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings); @@ -680,6 +680,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit); + scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_START_TIME, this::updateTimeSeriesStartTime); scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_END_TIME, this::updateTimeSeriesEndTime); } @@ -1164,18 +1165,32 @@ public long getTimeSeriesStartTime() { return timeSeriesStartTime; } + public void updateTimeSeriesStartTime(long startTime) { + if (timeSeriesStartTime < 0) { + throw new IllegalArgumentException("index.time_series.start_time not set before, can not update value"); + } + + if (this.timeSeriesStartTime < startTime) { + throw new IllegalArgumentException( + "index.time_series.start_time must be smaller than pre value [" + this.timeSeriesStartTime + "]" + ); + } + this.timeSeriesStartTime = startTime; + } + public long getTimeSeriesEndTime() { return timeSeriesEndTime; } - public void updateTimeSeriesEndTime(Instant endTime) { - long updateEndTime = endTime.toEpochMilli(); - if (this.timeSeriesEndTime > updateEndTime) { - throw new IllegalArgumentException( - "index.time_series.end_time must be larger than current value [" + this.timeSeriesEndTime + "]" - ); + public void updateTimeSeriesEndTime(long endTime) { + if (timeSeriesEndTime < 0) { + throw new IllegalArgumentException("index.time_series.end_time not set before, can not update value"); + } + + if (this.timeSeriesEndTime > endTime) { + throw new IllegalArgumentException("index.time_series.end_time must be larger than pre value [" + this.timeSeriesEndTime + "]"); } - this.timeSeriesEndTime = updateEndTime; + this.timeSeriesEndTime = endTime; } } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index c6fbbbdbe4b2c..5c70c3f904ec7 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.translog.Translog; @@ -22,7 +23,6 @@ import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; -import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -551,8 +551,8 @@ public void testCustomDataPathDeprecated() { } public void testUpdateTimeSeriesTimeRange() { - long endTime = System.currentTimeMillis(); - long startTime = endTime - TimeUnit.DAYS.toMillis(1); + long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000); + long endTime = startTime + randomLongBetween(1000, 86400000); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put(TIME_SERIES_START_TIME.getKey(), startTime) @@ -561,45 +561,81 @@ public void testUpdateTimeSeriesTimeRange() { IndexMetadata metadata = newIndexMeta("test", settings); IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - // test update end_time - // smaller - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(endTime - randomLongBetween(1, 1000))) - ); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than current value")); + // test update start_time + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesStartTime(startTime + randomLongBetween(10, 1000)) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time must be smaller than pre value")); + + // success + long newStartTime = Math.max(startTime - randomLongBetween(10, 1000), 0); + indexSettings.updateTimeSeriesStartTime(newStartTime); + assertEquals(newStartTime, indexSettings.getTimeSeriesStartTime()); + } - // success - long newEndTime = endTime + randomLongBetween(1, 1000); - indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(newEndTime)); - assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); + // test update end_time + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesEndTime(endTime - randomLongBetween(10, 1000)) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than pre value")); + + // success + long newEndTime = endTime + randomLongBetween(10, 1000); + indexSettings.updateTimeSeriesEndTime(newEndTime); + assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); + } } - public void testTimeSeriesTimeBoundary() { - long startTime = System.currentTimeMillis(); - long endTime = startTime - randomLongBetween(1, 1000); + public void testUpdateTimeSeriesTimeRangeNotSet() { final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) - .put(TIME_SERIES_START_TIME.getKey(), startTime) - .put(TIME_SERIES_END_TIME.getKey(), endTime) .build(); IndexMetadata metadata = newIndexMeta("test", settings); + IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than index.time_series.start_time")); + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesStartTime(randomLongBetween(10, 1000)) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time not set before, can not update value")); + } + + { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesEndTime(randomLongBetween(10, 1000)) + ); + + assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time not set before, can not update value")); + } } - public void testTimeSeriesTimeRangeModeNotSet() { - long time = System.currentTimeMillis(); + public void testTimeSeriesTimeRangeModeSet() { + long time = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999); { - final Settings settings = Settings.builder().put(TIME_SERIES_START_TIME.getKey(), time).build(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TIME_SERIES_START_TIME.get(settings)); + final Settings settings = Settings.builder() + .put(TIME_SERIES_START_TIME.getKey(), time) + .build(); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> TIME_SERIES_START_TIME.get(settings) + ); assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time need to be used for time_series mode")); } { - final Settings settings = Settings.builder().put(TIME_SERIES_END_TIME.getKey(), time).build(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TIME_SERIES_END_TIME.get(settings)); + final Settings settings = Settings.builder() + .put(TIME_SERIES_END_TIME.getKey(), time) + .build(); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> TIME_SERIES_END_TIME.get(settings) + ); assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time need to be used for time_series mode")); } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 7ab5906c10a2c..1e360714cd03e 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; @@ -19,7 +20,6 @@ import org.elasticsearch.index.mapper.SourceToParse; import java.io.IOException; -import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -62,9 +62,8 @@ public void testSortOrder() { } public void testValidateTimestamp() throws IOException { - long endTime = System.currentTimeMillis(); - long startTime = endTime - TimeUnit.DAYS.toMillis(1); - + long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000); + long endTime = startTime + randomLongBetween(1000, 86400000); Settings s = Settings.builder() .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) @@ -100,7 +99,7 @@ public void testValidateTimestamp() throws IOException { BytesReference.bytes( XContentFactory.jsonBuilder() .startObject() - .field("@timestamp", Math.max(startTime - randomLongBetween(1, 10), 0)) + .field("@timestamp", Math.max(startTime - randomLongBetween(10, 1000), 0)) .endObject() ), XContentType.JSON @@ -121,7 +120,7 @@ public void testValidateTimestamp() throws IOException { BytesReference.bytes( XContentFactory.jsonBuilder() .startObject() - .field("@timestamp", endTime + randomLongBetween(0, 10)) + .field("@timestamp", endTime + randomLongBetween(10, 1000)) .endObject() ), XContentType.JSON From 0e677263a3fce9fb08b25580835cd820d52dc6f2 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 28 Sep 2021 19:23:46 +0800 Subject: [PATCH 08/24] Revert "Revert "improve"" This reverts commit 63e552cbca89f94c41d7e603379fdbe90041eabe. --- .../rest-api-spec/test/tsdb/10_settings.yml | 12 +-- .../common/settings/Setting.java | 22 +++-- .../org/elasticsearch/index/IndexMode.java | 2 +- .../elasticsearch/index/IndexSettings.java | 69 ++++++-------- .../index/IndexSettingsTests.java | 90 ++++++------------- .../index/TimeSeriesModeTests.java | 11 +-- 6 files changed, 76 insertions(+), 130 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index 7ccc8aba525bd..fb6c4e14028c3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -140,20 +140,10 @@ set start_time and end_time: body: index: time_series: - start_time: 1632625781000 end_time: 1632625793000 - do: - catch: /index.time_series.start_time must be smaller than pre value \[1632625781000\]/ - indices.put_settings: - index: test_index - body: - index: - time_series: - start_time: 1632625783000 - - - do: - catch: /index.time_series.end_time must be larger than pre value \[1632625793000\]/ + catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/ indices.put_settings: index: test_index body: diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 5eb5a5f91ee98..be727b632ba0f 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -12,15 +12,11 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.MemorySizeValue; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -29,8 +25,14 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.mapper.DateFieldMapper; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -1222,10 +1224,14 @@ public static Setting longSetting(String key, long defaultValue, long minV properties); } - public static Setting longSetting(String key, long defaultValue, long minValue, Validator validator, - Property... properties) { - return new Setting<>(key, Long.toString(defaultValue), (s) -> parseLong(s, minValue, key, isFiltered(properties)), validator, - properties); + public static Setting dateSetting(String key, Instant defaultValue, Validator validator, Property... properties) { + return new Setting<>( + key, + defaultValue.toString(), + (s) -> Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(s)), + validator, + properties + ); } public static Setting simpleString(String key, Property... properties) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index e7866723b90c8..60356add0bb2b 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -83,7 +83,7 @@ private void validateTimestamp(DocumentParserContext context) { ); } - if (timestamp > endTime) { + if (timestamp >= endTime) { throw new IllegalArgumentException( "time series index @timestamp value [" + timestamp + "] must be smaller than " + endTime ); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 36559405d4c90..a08861cfe52ba 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Booleans; @@ -26,6 +27,7 @@ import org.elasticsearch.ingest.IngestService; import org.elasticsearch.node.Node; +import java.time.Instant; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -365,18 +367,16 @@ public Iterator> settings() { /** * in time series mode, the start time of the index, timestamp must larger than start_time - * format is epoch_millis */ - public static final Setting TIME_SERIES_START_TIME = Setting.longSetting( + public static final Setting TIME_SERIES_START_TIME = Setting.dateSetting( "index.time_series.start_time", - -1L, - -1L, + Instant.ofEpochMilli(0), new Setting.Validator<>() { @Override - public void validate(Long value) {} + public void validate(Instant value) {} @Override - public void validate(Long value, Map, Object> settings) { + public void validate(Instant value, Map, Object> settings) { IndexMode mode = (IndexMode) settings.get(MODE); if (mode != IndexMode.TIME_SERIES) { throw new IllegalArgumentException("index.time_series.start_time need to be used for time_series mode"); @@ -389,33 +389,35 @@ public Iterator> settings() { return settings.iterator(); } }, - Property.IndexScope, - Property.Dynamic + Property.IndexScope ); /** * in time series mode, the end time of the index, timestamp must smaller than start_time - * format is epoch_millis */ - public static final Setting TIME_SERIES_END_TIME = Setting.longSetting( + public static final Setting TIME_SERIES_END_TIME = Setting.dateSetting( "index.time_series.end_time", - -1L, - -1L, + DateUtils.MAX_NANOSECOND_INSTANT, new Setting.Validator<>() { @Override - public void validate(Long value) {} + public void validate(Instant value) {} @Override - public void validate(Long value, Map, Object> settings) { + public void validate(Instant value, Map, Object> settings) { IndexMode mode = (IndexMode) settings.get(MODE); if (mode != IndexMode.TIME_SERIES) { throw new IllegalArgumentException("index.time_series.end_time need to be used for time_series mode"); } + + Instant startTime = (Instant) settings.get(TIME_SERIES_START_TIME); + if (startTime.toEpochMilli() > value.toEpochMilli()) { + throw new IllegalArgumentException("index.time_series.end_time must be larger than index.time_series.start_time"); + } } @Override public Iterator> settings() { - final List> settings = List.of(MODE); + final List> settings = List.of(MODE, TIME_SERIES_START_TIME); return settings.iterator(); } }, @@ -433,7 +435,7 @@ public Iterator> settings() { * The {@link IndexMode "mode"} of the index. */ private final IndexMode mode; - private volatile long timeSeriesStartTime; + private final long timeSeriesStartTime; private volatile long timeSeriesEndTime; // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock @@ -577,10 +579,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD; - if (mode == IndexMode.TIME_SERIES) { - timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings); - timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings); - } + timeSeriesStartTime = mode == IndexMode.TIME_SERIES ? TIME_SERIES_START_TIME.get(settings).toEpochMilli() : -1L; + timeSeriesEndTime = mode == IndexMode.TIME_SERIES ? TIME_SERIES_END_TIME.get(settings).toEpochMilli() : -1L; this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings); this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings); @@ -680,7 +680,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit); - scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_START_TIME, this::updateTimeSeriesStartTime); scopedSettings.addSettingsUpdateConsumer(TIME_SERIES_END_TIME, this::updateTimeSeriesEndTime); } @@ -1165,32 +1164,18 @@ public long getTimeSeriesStartTime() { return timeSeriesStartTime; } - public void updateTimeSeriesStartTime(long startTime) { - if (timeSeriesStartTime < 0) { - throw new IllegalArgumentException("index.time_series.start_time not set before, can not update value"); - } - - if (this.timeSeriesStartTime < startTime) { - throw new IllegalArgumentException( - "index.time_series.start_time must be smaller than pre value [" + this.timeSeriesStartTime + "]" - ); - } - this.timeSeriesStartTime = startTime; - } - public long getTimeSeriesEndTime() { return timeSeriesEndTime; } - public void updateTimeSeriesEndTime(long endTime) { - if (timeSeriesEndTime < 0) { - throw new IllegalArgumentException("index.time_series.end_time not set before, can not update value"); - } - - if (this.timeSeriesEndTime > endTime) { - throw new IllegalArgumentException("index.time_series.end_time must be larger than pre value [" + this.timeSeriesEndTime + "]"); + public void updateTimeSeriesEndTime(Instant endTime) { + long updateEndTime = endTime.toEpochMilli(); + if (this.timeSeriesEndTime > updateEndTime) { + throw new IllegalArgumentException( + "index.time_series.end_time must be larger than current value [" + this.timeSeriesEndTime + "]" + ); } - this.timeSeriesEndTime = endTime; + this.timeSeriesEndTime = updateEndTime; } } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 5c70c3f904ec7..c6fbbbdbe4b2c 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.translog.Translog; @@ -23,6 +22,7 @@ import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -551,8 +551,8 @@ public void testCustomDataPathDeprecated() { } public void testUpdateTimeSeriesTimeRange() { - long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000); - long endTime = startTime + randomLongBetween(1000, 86400000); + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put(TIME_SERIES_START_TIME.getKey(), startTime) @@ -561,81 +561,45 @@ public void testUpdateTimeSeriesTimeRange() { IndexMetadata metadata = newIndexMeta("test", settings); IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - // test update start_time - { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesStartTime(startTime + randomLongBetween(10, 1000)) - ); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time must be smaller than pre value")); - - // success - long newStartTime = Math.max(startTime - randomLongBetween(10, 1000), 0); - indexSettings.updateTimeSeriesStartTime(newStartTime); - assertEquals(newStartTime, indexSettings.getTimeSeriesStartTime()); - } - // test update end_time - { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesEndTime(endTime - randomLongBetween(10, 1000)) - ); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than pre value")); - - // success - long newEndTime = endTime + randomLongBetween(10, 1000); - indexSettings.updateTimeSeriesEndTime(newEndTime); - assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); - } + // smaller + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(endTime - randomLongBetween(1, 1000))) + ); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than current value")); + + // success + long newEndTime = endTime + randomLongBetween(1, 1000); + indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(newEndTime)); + assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); } - public void testUpdateTimeSeriesTimeRangeNotSet() { + public void testTimeSeriesTimeBoundary() { + long startTime = System.currentTimeMillis(); + long endTime = startTime - randomLongBetween(1, 1000); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(TIME_SERIES_START_TIME.getKey(), startTime) + .put(TIME_SERIES_END_TIME.getKey(), endTime) .build(); IndexMetadata metadata = newIndexMeta("test", settings); - IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesStartTime(randomLongBetween(10, 1000)) - ); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time not set before, can not update value")); - } - - { - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesEndTime(randomLongBetween(10, 1000)) - ); - - assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time not set before, can not update value")); - } + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); + assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than index.time_series.start_time")); } - public void testTimeSeriesTimeRangeModeSet() { - long time = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999); + public void testTimeSeriesTimeRangeModeNotSet() { + long time = System.currentTimeMillis(); { - final Settings settings = Settings.builder() - .put(TIME_SERIES_START_TIME.getKey(), time) - .build(); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> TIME_SERIES_START_TIME.get(settings) - ); + final Settings settings = Settings.builder().put(TIME_SERIES_START_TIME.getKey(), time).build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TIME_SERIES_START_TIME.get(settings)); assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time need to be used for time_series mode")); } { - final Settings settings = Settings.builder() - .put(TIME_SERIES_END_TIME.getKey(), time) - .build(); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> TIME_SERIES_END_TIME.get(settings) - ); + final Settings settings = Settings.builder().put(TIME_SERIES_END_TIME.getKey(), time).build(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TIME_SERIES_END_TIME.get(settings)); assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time need to be used for time_series mode")); } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 1e360714cd03e..7ab5906c10a2c 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; @@ -20,6 +19,7 @@ import org.elasticsearch.index.mapper.SourceToParse; import java.io.IOException; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -62,8 +62,9 @@ public void testSortOrder() { } public void testValidateTimestamp() throws IOException { - long startTime = Math.min(randomMillisUpToYear9999(), DateUtils.MAX_MILLIS_BEFORE_9999 - 86400000); - long endTime = startTime + randomLongBetween(1000, 86400000); + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + Settings s = Settings.builder() .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) @@ -99,7 +100,7 @@ public void testValidateTimestamp() throws IOException { BytesReference.bytes( XContentFactory.jsonBuilder() .startObject() - .field("@timestamp", Math.max(startTime - randomLongBetween(10, 1000), 0)) + .field("@timestamp", Math.max(startTime - randomLongBetween(1, 10), 0)) .endObject() ), XContentType.JSON @@ -120,7 +121,7 @@ public void testValidateTimestamp() throws IOException { BytesReference.bytes( XContentFactory.jsonBuilder() .startObject() - .field("@timestamp", endTime + randomLongBetween(10, 1000)) + .field("@timestamp", endTime + randomLongBetween(0, 10)) .endObject() ), XContentType.JSON From 78c132d929f3030b142d363f714679f801125469 Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 18 Oct 2021 17:42:48 +0800 Subject: [PATCH 09/24] spotless --- .../common/settings/Setting.java | 22 +++++++++---------- .../org/elasticsearch/index/IndexMode.java | 3 +-- .../index/TimeSeriesModeTests.java | 5 +---- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 6532b126ba729..230a4e91138a8 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -12,24 +12,24 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.core.Nullable; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.MemorySizeValue; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParserUtils; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.xcontent.DeprecationHandler; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.time.Instant; diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 40c25c32da900..be1877118f355 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -13,13 +13,12 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentParserContext; -import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.RoutingFieldMapper; - import java.util.Arrays; import java.util.List; import java.util.Map; diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 4644236abb258..966e5c0a4f0f1 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -128,10 +128,7 @@ public void testValidateTimestamp() throws IOException { "test", "1", BytesReference.bytes( - XContentFactory.jsonBuilder() - .startObject() - .field("@timestamp", endTime + randomLongBetween(0, 10)) - .endObject() + XContentFactory.jsonBuilder().startObject().field("@timestamp", endTime + randomLongBetween(0, 10)).endObject() ), XContentType.JSON ) From 120cdaf97b6d7001eae782de8bb3a3fffa9045b2 Mon Sep 17 00:00:00 2001 From: weizijun Date: Wed, 27 Oct 2021 11:29:57 +0800 Subject: [PATCH 10/24] improve --- .../rest-api-spec/test/tsdb/10_settings.yml | 62 ------ .../test/tsdb/15_timestamp_mapping.yml | 62 ++++++ .../org/elasticsearch/index/IndexMode.java | 40 ++-- .../DataStreamTimestampFieldMapper.java | 7 + .../index/mapper/DocumentParser.java | 2 - .../index/IndexSettingsTests.java | 2 + .../index/TimeSeriesModeTests.java | 188 ++++++++++-------- 7 files changed, 190 insertions(+), 173 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index 0ebe3e440061b..861b3292bd520 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -119,68 +119,6 @@ no partitioning: shards: 5 routing_partition_size: 2 ---- -set start_time and end_time: - - skip: - version: " - 7.99.99" - reason: introduced in 8.0.0 - - do: - indices.create: - index: test_index - body: - settings: - index: - mode: time_series - time_series: - start_time: 1632625782000 - end_time: 1632625792000 - - - do: - indices.put_settings: - index: test_index - body: - index: - time_series: - end_time: 1632625793000 - - - do: - catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/ - indices.put_settings: - index: test_index - body: - index: - time_series: - end_time: 1632625792000 - - - do: - indices.delete: - index: test_index - ---- -set start_time and end_time without timeseries mode: - - skip: - version: " - 7.99.99" - reason: introduced in 8.0.0 - - do: - catch: /index.time_series.start_time need to be used for time_series mode/ - indices.create: - index: test_index - body: - settings: - index: - time_series: - start_time: 1632625782000 - - - do: - catch: /index.time_series.end_time need to be used for time_series mode/ - indices.create: - index: test_index - body: - settings: - index: - time_series: - end_time: 1632625782000 - --- routing_path required: - skip: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml index 9d61d4c359b6d..6cfe0a9e82758 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -177,3 +177,65 @@ reject timestamp meta field with wrong type: mappings: _data_stream_timestamp: enabled: false +--- +set start_time and end_time: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + - do: + indices.create: + index: test_index + body: + settings: + index: + mode: time_series + routing_path: [metricset] + time_series: + start_time: 1632625782000 + end_time: 1632625792000 + + - do: + indices.put_settings: + index: test_index + body: + index: + time_series: + end_time: 1632625793000 + + - do: + catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/ + indices.put_settings: + index: test_index + body: + index: + time_series: + end_time: 1632625792000 + + - do: + indices.delete: + index: test_index + +--- +set start_time and end_time without timeseries mode: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + - do: + catch: /index.time_series.start_time need to be used for time_series mode/ + indices.create: + index: test_index + body: + settings: + index: + time_series: + start_time: 1632625782000 + + - do: + catch: /index.time_series.end_time need to be used for time_series mode/ + indices.create: + index: test_index + body: + settings: + index: + time_series: + end_time: 1632625782000 diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 16ba3d8d7b020..91ffd3d7d5aa4 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -8,23 +8,20 @@ package org.elasticsearch.index; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.IndexableField; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentParserContext; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,8 +38,6 @@ public enum IndexMode { STANDARD { @Override - public void validateWithSource(DocumentParserContext context) {} - void validateWithOtherSettings(Map, Object> settings) { if (false == Objects.equals( IndexMetadata.INDEX_ROUTING_PATH.getDefault(Settings.EMPTY), @@ -62,6 +57,9 @@ public void validateAlias(@Nullable String indexRouting, @Nullable String search @Override public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) {} + + @Override + public void validateTimestampRange(DocumentParserContext context, long timestamp) {} }, TIME_SERIES { public static final String TIMESTAMP_FIELD = "@timestamp"; @@ -84,24 +82,7 @@ void validateWithOtherSettings(Map, Object> settings) { } @Override - public void validateWithSource(DocumentParserContext context) { - validateTimestamp(context); - } - - private void validateTimestamp(DocumentParserContext context) { - IndexableField[] fields = context.rootDoc().getFields(TIMESTAMP_FIELD); - if (fields.length == 0) { - throw new IllegalArgumentException("time series index @timestamp field is missing"); - } - - long numberOfValues = Arrays.stream(fields) - .filter(indexableField -> indexableField.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) - .count(); - if (numberOfValues > 1) { - throw new IllegalArgumentException("time series index @timestamp field encountered multiple values"); - } - - long timestamp = fields[0].numericValue().longValue(); + public void validateTimestampRange(DocumentParserContext context, long timestamp) { if (context.mappingLookup().getMapper(TIMESTAMP_FIELD).typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) { timestamp /= NSEC_PER_MSEC; } @@ -111,13 +92,13 @@ private void validateTimestamp(DocumentParserContext context) { if (timestamp < startTime) { throw new IllegalArgumentException( - "time series index @timestamp value [" + timestamp + "] must be larger than " + startTime + "time series index " + TIMESTAMP_FIELD + " value [" + timestamp + "] must be larger than " + startTime ); } if (timestamp >= endTime) { throw new IllegalArgumentException( - "time series index @timestamp value [" + timestamp + "] must be smaller than " + endTime + "time series index " + TIMESTAMP_FIELD + " value [" + timestamp + "] must be smaller than " + endTime ); } } @@ -206,8 +187,6 @@ private void validateTimeStampField(Object timestampFieldValue) { abstract void validateWithOtherSettings(Map, Object> settings); - public abstract void validateWithSource(DocumentParserContext context); - /** * Validate the mapping for this index. */ @@ -222,4 +201,9 @@ private void validateTimeStampField(Object timestampFieldValue) { * Validate and/or modify the mappings after after they've been parsed. */ public abstract void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder); + + /** + * Validate the timestamp, check the timestamp range between start_time and end_time + */ + public abstract void validateTimestampRange(DocumentParserContext context, long timestamp); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 616c10b052408..3d629797c828e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.xcontent.XContentType; @@ -209,6 +210,12 @@ public void postParse(DocumentParserContext context) throws IOException { if (numberOfValues > 1) { throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] encountered multiple values"); } + + IndexMode mode = context.indexSettings().getMode(); + if (mode == IndexMode.TIME_SERIES) { + long timestamp = fields[0].numericValue().longValue(); + context.indexSettings().getMode().validateTimestampRange(context, timestamp); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 6d86f6cb717d7..03611bc8ae074 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -140,8 +140,6 @@ private static void internalParseDocument(RootObjectMapper root, MetadataFieldMa executeIndexTimeScripts(context); - context.indexSettings().getMode().validateWithSource(context); - for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) { metadataMapper.postParse(context); } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index c6fbbbdbe4b2c..9e92a2b727c7f 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -555,6 +555,7 @@ public void testUpdateTimeSeriesTimeRange() { long startTime = endTime - TimeUnit.DAYS.toMillis(1); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") .put(TIME_SERIES_START_TIME.getKey(), startTime) .put(TIME_SERIES_END_TIME.getKey(), endTime) .build(); @@ -580,6 +581,7 @@ public void testTimeSeriesTimeBoundary() { long endTime = startTime - randomLongBetween(1, 1000); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") .put(TIME_SERIES_START_TIME.getKey(), startTime) .put(TIME_SERIES_END_TIME.getKey(), endTime) .build(); diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index a4ee20973c440..b47f8f25e5132 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -83,87 +83,6 @@ public void testSortOrder() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); } - public void testValidateTimestamp() throws IOException { - long endTime = System.currentTimeMillis(); - long startTime = endTime - TimeUnit.DAYS.toMillis(1); - - Settings s = Settings.builder() - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) - .put(IndexSettings.MODE.getKey(), "time_series") - .build(); - - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); - // timestamp missing - { - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse( - new SourceToParse( - "test", - "1", - BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()), - XContentType.JSON - ) - ) - ); - assertThat(e.getRootCause().getMessage(), containsString("time series index @timestamp field is missing")); - } - - // timestamp smaller than start_time - { - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse( - new SourceToParse( - "test", - "1", - BytesReference.bytes( - XContentFactory.jsonBuilder() - .startObject() - .field("@timestamp", Math.max(startTime - randomLongBetween(1, 10), 0)) - .endObject() - ), - XContentType.JSON - ) - ) - ); - assertThat(e.getRootCause().getMessage(), containsString("must be larger than")); - } - - // timestamp larger than end_time - { - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse( - new SourceToParse( - "test", - "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("@timestamp", endTime + randomLongBetween(0, 10)).endObject() - ), - XContentType.JSON - ) - ) - ); - assertThat(e.getRootCause().getMessage(), containsString("must be smaller than")); - } - - // timestamp correct - { - mapper.parse( - new SourceToParse( - "test", - "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("@timestamp", randomLongBetween(startTime, endTime)).endObject() - ), - XContentType.JSON - ) - ); - } - public void testAddsTimestamp() throws IOException { Settings s = Settings.builder() .put(IndexSettings.MODE.getKey(), "time_series") @@ -227,6 +146,113 @@ public void testBadTimestamp() throws IOException { ); } + public void testWithoutTimestamp() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) + .documentMapper(); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()), + XContentType.JSON + ) + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("data stream timestamp field [@timestamp] is missing")); + } + + public void testEnableTimestampRange() throws IOException { + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) + .documentMapper(); + mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("@timestamp", randomLongBetween(startTime, endTime)).endObject() + ), + XContentType.JSON + ) + ); + } + + public void testBadStartTime() throws IOException { + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) + .documentMapper(); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .field("@timestamp", Math.max(startTime - randomLongBetween(1, 3), 0)) + .endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("must be larger than")); + } + + public void testBadEndTime() throws IOException { + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) + .documentMapper(); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> mapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().field("@timestamp", endTime + randomLongBetween(0, 3)).endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("must be smaller than")); + } + public void testEnabledTimeStampMapper() throws IOException { Settings s = Settings.builder() .put(IndexSettings.MODE.getKey(), "time_series") From 563ca874c04b01758234ce5395391f60a659c8e2 Mon Sep 17 00:00:00 2001 From: weizijun Date: Wed, 27 Oct 2021 11:40:15 +0800 Subject: [PATCH 11/24] fix ccr tests --- .../xpack/ccr/action/TransportResumeFollowActionTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 561015a5888be..bec957c448fba 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -219,6 +219,7 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); replicatedSettings.add(IndexSettings.MAX_NGRAM_DIFF_SETTING); replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); + replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { if (setting.isDynamic()) { From 14a2378190535aa1ef5b74e4a8968d6897ce43aa Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 28 Oct 2021 00:31:17 +0800 Subject: [PATCH 12/24] spotless --- .../index/mapper/DataStreamTimestampFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index c48d50ecfde39..00f77073b3647 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -13,8 +13,8 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; From 2c45c5721545ce9b6c5941bf4c459146b8cb2fd1 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 28 Oct 2021 10:20:37 +0800 Subject: [PATCH 13/24] improve --- .../rest-api-spec/test/tsdb/10_settings.yml | 62 +++++++++++++++++++ .../test/tsdb/15_timestamp_mapping.yml | 62 ------------------- .../elasticsearch/index/IndexSettings.java | 8 ++- .../DataStreamTimestampFieldMapper.java | 2 +- 4 files changed, 69 insertions(+), 65 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index 861b3292bd520..54bd643b45b74 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -168,3 +168,65 @@ routing required: mappings: _routing: required: true +--- +set start_time and end_time: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + - do: + indices.create: + index: test_index + body: + settings: + index: + mode: time_series + routing_path: [metricset] + time_series: + start_time: 1632625782000 + end_time: 1632625792000 + + - do: + indices.put_settings: + index: test_index + body: + index: + time_series: + end_time: 1632625793000 + + - do: + catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/ + indices.put_settings: + index: test_index + body: + index: + time_series: + end_time: 1632625792000 + + - do: + indices.delete: + index: test_index + +--- +set start_time and end_time without timeseries mode: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + - do: + catch: /index.time_series.start_time need to be used for time_series mode/ + indices.create: + index: test_index + body: + settings: + index: + time_series: + start_time: 1632625782000 + + - do: + catch: /index.time_series.end_time need to be used for time_series mode/ + indices.create: + index: test_index + body: + settings: + index: + time_series: + end_time: 1632625782000 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml index 6cfe0a9e82758..9d61d4c359b6d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -177,65 +177,3 @@ reject timestamp meta field with wrong type: mappings: _data_stream_timestamp: enabled: false ---- -set start_time and end_time: - - skip: - version: " - 7.99.99" - reason: introduced in 8.0.0 - - do: - indices.create: - index: test_index - body: - settings: - index: - mode: time_series - routing_path: [metricset] - time_series: - start_time: 1632625782000 - end_time: 1632625792000 - - - do: - indices.put_settings: - index: test_index - body: - index: - time_series: - end_time: 1632625793000 - - - do: - catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/ - indices.put_settings: - index: test_index - body: - index: - time_series: - end_time: 1632625792000 - - - do: - indices.delete: - index: test_index - ---- -set start_time and end_time without timeseries mode: - - skip: - version: " - 7.99.99" - reason: introduced in 8.0.0 - - do: - catch: /index.time_series.start_time need to be used for time_series mode/ - indices.create: - index: test_index - body: - settings: - index: - time_series: - start_time: 1632625782000 - - - do: - catch: /index.time_series.end_time need to be used for time_series mode/ - indices.create: - index: test_index - body: - settings: - index: - time_series: - end_time: 1632625782000 diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index a57a0e4b86ba3..e92e871afb509 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -715,8 +715,12 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD; - timeSeriesStartTime = mode == IndexMode.TIME_SERIES ? TIME_SERIES_START_TIME.get(settings).toEpochMilli() : -1L; - timeSeriesEndTime = mode == IndexMode.TIME_SERIES ? TIME_SERIES_END_TIME.get(settings).toEpochMilli() : -1L; + timeSeriesStartTime = mode == IndexMode.TIME_SERIES + ? TIME_SERIES_START_TIME.get(settings).toEpochMilli() + : Instant.ofEpochMilli(0).toEpochMilli(); + timeSeriesEndTime = mode == IndexMode.TIME_SERIES + ? TIME_SERIES_END_TIME.get(settings).toEpochMilli() + : DateUtils.MAX_NANOSECOND_INSTANT.toEpochMilli(); this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings); this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 00f77073b3647..39e2aba352638 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -213,7 +213,7 @@ public void postParse(DocumentParserContext context) throws IOException { } IndexMode mode = context.indexSettings().getMode(); - if (mode == IndexMode.TIME_SERIES) { + if (mode != null && mode == IndexMode.TIME_SERIES) { long timestamp = fields[0].numericValue().longValue(); context.indexSettings().getMode().validateTimestampRange(context, timestamp); } From 1dcebb08382fd519d1aa9a0d7b2000b1b4d61c6b Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 2 Nov 2021 14:27:56 +0800 Subject: [PATCH 14/24] fix bwc --- .../resources/rest-api-spec/test/tsdb/10_settings.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index 54bd643b45b74..d03f62b89e70c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -171,8 +171,8 @@ routing required: --- set start_time and end_time: - skip: - version: " - 7.99.99" - reason: introduced in 8.0.0 + version: " - 8.0.99" + reason: introduced in 8.1.0 - do: indices.create: index: test_index @@ -209,8 +209,8 @@ set start_time and end_time: --- set start_time and end_time without timeseries mode: - skip: - version: " - 7.99.99" - reason: introduced in 8.0.0 + version: " - 8.0.99" + reason: introduced in 8.1.0 - do: catch: /index.time_series.start_time need to be used for time_series mode/ indices.create: From 79db8821191be59eca695b134ceb9e5e7fe1a03e Mon Sep 17 00:00:00 2001 From: weizijun Date: Wed, 3 Nov 2021 23:41:37 +0800 Subject: [PATCH 15/24] improve --- .../org/elasticsearch/index/IndexMode.java | 25 +++++++++++-------- .../elasticsearch/index/IndexSettings.java | 6 +++++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 8e8e5a4ca7145..ef9370fc449c5 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -87,19 +87,22 @@ public void validateTimestampRange(DocumentParserContext context, long timestamp timestamp /= NSEC_PER_MSEC; } - long startTime = context.indexSettings().getTimeSeriesStartTime(); - long endTime = context.indexSettings().getTimeSeriesEndTime(); - - if (timestamp < startTime) { - throw new IllegalArgumentException( - "time series index " + TIMESTAMP_FIELD + " value [" + timestamp + "] must be larger than " + startTime - ); + if (IndexSettings.TIME_SERIES_START_TIME.get(context.indexSettings().getSettings()) != null) { + Long startTime = context.indexSettings().getTimeSeriesStartTime(); + if (timestamp < startTime) { + throw new IllegalArgumentException( + "time series index " + TIMESTAMP_FIELD + " value [" + timestamp + "] must be larger than " + startTime + ); + } } - if (timestamp >= endTime) { - throw new IllegalArgumentException( - "time series index " + TIMESTAMP_FIELD + " value [" + timestamp + "] must be smaller than " + endTime - ); + if (IndexSettings.TIME_SERIES_END_TIME.get(context.indexSettings().getSettings()) != null) { + Long endTime = context.indexSettings().getTimeSeriesEndTime(); + if (timestamp >= endTime) { + throw new IllegalArgumentException( + "time series index " + TIMESTAMP_FIELD + " value [" + timestamp + "] must be smaller than " + endTime + ); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index e92e871afb509..82c5dca0380b4 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -571,7 +571,13 @@ public Iterator> settings() { * The {@link IndexMode "mode"} of the index. */ private final IndexMode mode; + /** + * Start time of the time_series index. + */ private final long timeSeriesStartTime; + /** + * End time of the time_series index. + */ private volatile long timeSeriesEndTime; // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock From 69df14d2541d02e1630f7773f6c8272663d67cf7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 3 Nov 2021 14:04:00 -0400 Subject: [PATCH 16/24] Maybe like this? --- .../common/settings/Setting.java | 14 ++- .../org/elasticsearch/index/IndexMode.java | 71 ++++------- .../elasticsearch/index/IndexSettings.java | 118 ++++++++---------- .../DataStreamTimestampFieldMapper.java | 6 - .../index/mapper/DateFieldMapper.java | 39 +++++- 5 files changed, 124 insertions(+), 124 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 57a6e0e3a4a19..2bef52ef2b8d7 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -43,6 +43,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BiFunction; @@ -1333,11 +1334,16 @@ public static Setting longSetting(String key, long defaultValue, long minV ); } - public static Setting dateSetting(String key, Instant defaultValue, Validator validator, Property... properties) { - return new Setting<>( + public static Setting> dateSetting( + String key, + Optional defaultValue, + Validator> validator, + Property... properties + ) { + return new Setting>( key, - defaultValue.toString(), - (s) -> Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(s)), + defaultValue.map(Instant::toString).orElse(null), + s -> Optional.ofNullable(s).map(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER::parse).map(Instant::from), validator, properties ); diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index ef9370fc449c5..00eaec1444131 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -15,7 +15,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MappingParserContext; @@ -30,22 +29,26 @@ import java.util.stream.Stream; import static java.util.stream.Collectors.toSet; -import static org.elasticsearch.core.TimeValue.NSEC_PER_MSEC; /** * "Mode" that controls which behaviors and settings an index supports. + *

+ * For the most part this class concentrates on validating settings and + * mappings. Most different behavior is controlled by forcing settings + * to be set or not set and by enabling extra fields in the mapping. */ public enum IndexMode { STANDARD { @Override void validateWithOtherSettings(Map, Object> settings) { - if (false == Objects.equals( - IndexMetadata.INDEX_ROUTING_PATH.getDefault(Settings.EMPTY), - settings.get(IndexMetadata.INDEX_ROUTING_PATH) - )) { - throw new IllegalArgumentException( - "[" + IndexMetadata.INDEX_ROUTING_PATH.getKey() + "] requires [" + IndexSettings.MODE.getKey() + "=time_series]" - ); + settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH); + settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME); + settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_END_TIME); + } + + private void settingRequiresTimeSeries(Map, Object> settings, Setting setting) { + if (false == Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting))) { + throw new IllegalArgumentException("[" + setting.getKey() + "] requires [" + IndexSettings.MODE.getKey() + "=time_series]"); } } @@ -57,13 +60,8 @@ public void validateAlias(@Nullable String indexRouting, @Nullable String search @Override public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) {} - - @Override - public void validateTimestampRange(DocumentParserContext context, long timestamp) {} }, TIME_SERIES { - public static final String TIMESTAMP_FIELD = "@timestamp"; - @Override void validateWithOtherSettings(Map, Object> settings) { if (settings.get(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING) != Integer.valueOf(1)) { @@ -74,35 +72,14 @@ void validateWithOtherSettings(Map, Object> settings) { throw new IllegalArgumentException(error(unsupported)); } } - if (IndexMetadata.INDEX_ROUTING_PATH.getDefault(Settings.EMPTY).equals(settings.get(IndexMetadata.INDEX_ROUTING_PATH))) { - throw new IllegalArgumentException( - "[" + IndexSettings.MODE.getKey() + "=time_series] requires [" + IndexMetadata.INDEX_ROUTING_PATH.getKey() + "]" - ); - } + settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH); + settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME); + settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_END_TIME); } - @Override - public void validateTimestampRange(DocumentParserContext context, long timestamp) { - if (context.mappingLookup().getMapper(TIMESTAMP_FIELD).typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) { - timestamp /= NSEC_PER_MSEC; - } - - if (IndexSettings.TIME_SERIES_START_TIME.get(context.indexSettings().getSettings()) != null) { - Long startTime = context.indexSettings().getTimeSeriesStartTime(); - if (timestamp < startTime) { - throw new IllegalArgumentException( - "time series index " + TIMESTAMP_FIELD + " value [" + timestamp + "] must be larger than " + startTime - ); - } - } - - if (IndexSettings.TIME_SERIES_END_TIME.get(context.indexSettings().getSettings()) != null) { - Long endTime = context.indexSettings().getTimeSeriesEndTime(); - if (timestamp >= endTime) { - throw new IllegalArgumentException( - "time series index " + TIMESTAMP_FIELD + " value [" + timestamp + "] must be smaller than " + endTime - ); - } + private void settingRequiresTimeSeries(Map, Object> settings, Setting setting) { + if (Objects.equals(setting.getDefault(Settings.EMPTY), settings.get(setting))) { + throw new IllegalArgumentException("[" + IndexSettings.MODE.getKey() + "=time_series] requires [" + setting.getKey() + "]"); } } @@ -182,7 +159,12 @@ private void validateTimeStampField(Object timestampFieldValue) { static final List> VALIDATE_WITH_SETTINGS = List.copyOf( Stream.concat( - Stream.of(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, IndexMetadata.INDEX_ROUTING_PATH), + Stream.of( + IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING, + IndexMetadata.INDEX_ROUTING_PATH, + IndexSettings.TIME_SERIES_START_TIME, + IndexSettings.TIME_SERIES_END_TIME + ), TIME_SERIES_UNSUPPORTED.stream() ).collect(toSet()) ); @@ -203,9 +185,4 @@ private void validateTimeStampField(Object timestampFieldValue) { * Validate and/or modify the mappings after after they've been parsed. */ public abstract void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder); - - /** - * Validate the timestamp, check the timestamp range between start_time and end_time - */ - public abstract void validateTimestampRange(DocumentParserContext context, long timestamp); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 82c5dca0380b4..ed3070e2eab55 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Booleans; @@ -32,6 +31,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; @@ -477,88 +477,77 @@ public static boolean isTimeSeriesModeEnabled() { } /** - * The {@link IndexMode "mode"} of the index. + * in time series mode, the start time of the index, timestamp must larger than start_time */ - public static final Setting MODE = Setting.enumSetting( - IndexMode.class, - "index.mode", - IndexMode.STANDARD, - new Setting.Validator() { - @Override - public void validate(IndexMode value) {} - - @Override - public void validate(IndexMode value, Map, Object> settings) { - value.validateWithOtherSettings(settings); - } - - @Override - public Iterator> settings() { - return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); - } - }, + public static final Setting> TIME_SERIES_START_TIME = Setting.dateSetting( + "index.time_series.start_time", + Optional.empty(), + v -> {}, Property.IndexScope, Property.Final ); /** - * in time series mode, the start time of the index, timestamp must larger than start_time + * in time series mode, the end time of the index, timestamp must smaller than start_time */ - public static final Setting TIME_SERIES_START_TIME = Setting.dateSetting( - "index.time_series.start_time", - Instant.ofEpochMilli(0), + public static final Setting> TIME_SERIES_END_TIME = Setting.dateSetting( + "index.time_series.end_time", + Optional.empty(), new Setting.Validator<>() { @Override - public void validate(Instant value) {} + public void validate(Optional value) {} @Override - public void validate(Instant value, Map, Object> settings) { - IndexMode mode = (IndexMode) settings.get(MODE); - if (mode != IndexMode.TIME_SERIES) { - throw new IllegalArgumentException("index.time_series.start_time need to be used for time_series mode"); + public void validate(Optional value, Map, Object> settings) { + @SuppressWarnings("unchecked") + Optional startTime = (Optional) settings.get(TIME_SERIES_START_TIME); + if (value.isEmpty()) { + if (startTime.isPresent()) { + throw new IllegalArgumentException("index.time_series.start_time must be set with index.time_series.end_time"); + } + return; + } + if (startTime.isEmpty()) { + throw new IllegalArgumentException("index.time_series.start_time must be set with index.time_series.end_time"); + } + if (startTime.get().compareTo(value.get()) < 0) { + throw new IllegalArgumentException("index.time_series.start_time must be larger than index.time_series.end_time"); } } @Override public Iterator> settings() { - final List> settings = List.of(MODE); + List> settings = List.of(TIME_SERIES_START_TIME); return settings.iterator(); } }, - Property.IndexScope + Property.IndexScope, + Property.Dynamic ); /** - * in time series mode, the end time of the index, timestamp must smaller than start_time + * The {@link IndexMode "mode"} of the index. */ - public static final Setting TIME_SERIES_END_TIME = Setting.dateSetting( - "index.time_series.end_time", - DateUtils.MAX_NANOSECOND_INSTANT, - new Setting.Validator<>() { + public static final Setting MODE = Setting.enumSetting( + IndexMode.class, + "index.mode", + IndexMode.STANDARD, + new Setting.Validator() { @Override - public void validate(Instant value) {} + public void validate(IndexMode value) {} @Override - public void validate(Instant value, Map, Object> settings) { - IndexMode mode = (IndexMode) settings.get(MODE); - if (mode != IndexMode.TIME_SERIES) { - throw new IllegalArgumentException("index.time_series.end_time need to be used for time_series mode"); - } - - Instant startTime = (Instant) settings.get(TIME_SERIES_START_TIME); - if (startTime.toEpochMilli() > value.toEpochMilli()) { - throw new IllegalArgumentException("index.time_series.end_time must be larger than index.time_series.start_time"); - } + public void validate(IndexMode value, Map, Object> settings) { + value.validateWithOtherSettings(settings); } @Override public Iterator> settings() { - final List> settings = List.of(MODE, TIME_SERIES_START_TIME); - return settings.iterator(); + return IndexMode.VALIDATE_WITH_SETTINGS.iterator(); } }, Property.IndexScope, - Property.Dynamic + Property.Final ); private final Index index; @@ -574,11 +563,11 @@ public Iterator> settings() { /** * Start time of the time_series index. */ - private final long timeSeriesStartTime; + private final Instant timeSeriesStartTime; /** * End time of the time_series index. */ - private volatile long timeSeriesEndTime; + private volatile Instant timeSeriesEndTime; // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock private volatile Settings settings; @@ -721,12 +710,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD; - timeSeriesStartTime = mode == IndexMode.TIME_SERIES - ? TIME_SERIES_START_TIME.get(settings).toEpochMilli() - : Instant.ofEpochMilli(0).toEpochMilli(); - timeSeriesEndTime = mode == IndexMode.TIME_SERIES - ? TIME_SERIES_END_TIME.get(settings).toEpochMilli() - : DateUtils.MAX_NANOSECOND_INSTANT.toEpochMilli(); + timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings).orElse(null); + timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings).orElse(null); this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings); this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings); @@ -1356,22 +1341,25 @@ private void setMappingDimensionFieldsLimit(long value) { this.mappingDimensionFieldsLimit = value; } - public long getTimeSeriesStartTime() { + public Instant getTimeSeriesStartTime() { return timeSeriesStartTime; } - public long getTimeSeriesEndTime() { + public Instant getTimeSeriesEndTime() { return timeSeriesEndTime; } - public void updateTimeSeriesEndTime(Instant endTime) { - long updateEndTime = endTime.toEpochMilli(); - if (this.timeSeriesEndTime > updateEndTime) { + public void updateTimeSeriesEndTime(Optional endTime) { + if (endTime.isEmpty()) { + this.timeSeriesEndTime = null; + return; + } + Instant endInstant = endTime.get(); + if (this.timeSeriesEndTime.isAfter(endInstant)) { throw new IllegalArgumentException( "index.time_series.end_time must be larger than current value [" + this.timeSeriesEndTime + "]" ); } - - this.timeSeriesEndTime = updateEndTime; + this.timeSeriesEndTime = endInstant; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 39e2aba352638..056013ac0f8d7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -211,12 +211,6 @@ public void postParse(DocumentParserContext context) throws IOException { if (numberOfValues > 1) { throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] encountered multiple values"); } - - IndexMode mode = context.indexSettings().getMode(); - if (mode != null && mode == IndexMode.TIME_SERIES) { - long timestamp = fields[0].numericValue().longValue(); - context.indexSettings().getMode().validateTimestampRange(context, timestamp); - } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 4e07b9bcfb1b1..b9d2bb034a9f0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -201,6 +201,22 @@ NumericType numericType() { */ public abstract long roundUpToMillis(long value); + public final void validateTimestamp(long value, DocumentParserContext context) { + if (context.indexSettings().getTimeSeriesStartTime() == null) { + assert context.indexSettings().getTimeSeriesEndTime() == null; + return; + } + long startTime = convert(context.indexSettings().getTimeSeriesStartTime()); + if (value < startTime) { + throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be larger than " + startTime); + } + + long endTime = convert(context.indexSettings().getTimeSeriesStartTime()); + if (value >= endTime) { + throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be smaller than " + endTime); + } + } + public static Resolution ofOrdinal(int ord) { for (Resolution resolution : values()) { if (ord == resolution.ordinal()) { @@ -244,6 +260,7 @@ public static class Builder extends FieldMapper.Builder { private final Resolution resolution; private final Version indexCreatedVersion; private final ScriptCompiler scriptCompiler; + private final Validate validate; public Builder( String name, @@ -270,6 +287,7 @@ public Builder( this.format.setValue(dateFormatter.pattern()); this.locale.setValue(dateFormatter.locale()); } + this.validate = name.equals(DataStreamTimestampFieldMapper.DEFAULT_PATH) ? resolution::validateTimestamp : (v, c) -> {}; } private DateFormatter buildFormatter() { @@ -336,7 +354,16 @@ public DateFieldMapper build(MapperBuilderContext context) { ); Long nullTimestamp = parseNullValue(ft); - return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); + return new DateFieldMapper( + name, + ft, + multiFieldsBuilder.build(this, context), + copyTo.build(), + nullTimestamp, + resolution, + this, + validate + ); } } @@ -716,6 +743,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { private final Script script; private final ScriptCompiler scriptCompiler; private final FieldValues scriptValues; + private final Validate validate; private DateFieldMapper( String simpleName, @@ -724,7 +752,8 @@ private DateFieldMapper( CopyTo copyTo, Long nullValue, Resolution resolution, - Builder builder + Builder builder, + Validate validate ) { super(simpleName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.get()); this.store = builder.store.getValue(); @@ -741,6 +770,7 @@ private DateFieldMapper( this.script = builder.script.get(); this.scriptCompiler = builder.scriptCompiler; this.scriptValues = builder.scriptValues(); + this.validate = validate; } @Override @@ -780,6 +810,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } } } + validate.validate(timestamp, context); indexValue(context, timestamp); } @@ -815,4 +846,8 @@ public boolean getIgnoreMalformed() { public Long getNullValue() { return nullValue; } + + private interface Validate { + void validate(long value, DocumentParserContext context); + } } From 6c77b189f573f1c16a7edf69e4f74cfed338ad01 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 11 Nov 2021 21:30:30 +0800 Subject: [PATCH 17/24] fix failed tests --- server/src/main/java/org/elasticsearch/index/IndexMode.java | 4 +--- .../java/org/elasticsearch/index/IndexSettingsTests.java | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 00eaec1444131..e6103eef98372 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -35,7 +35,7 @@ *

* For the most part this class concentrates on validating settings and * mappings. Most different behavior is controlled by forcing settings - * to be set or not set and by enabling extra fields in the mapping. + * to be set or not set and by enabling extra fields in the mapping. */ public enum IndexMode { STANDARD { @@ -73,8 +73,6 @@ void validateWithOtherSettings(Map, Object> settings) { } } settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH); - settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME); - settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_END_TIME); } private void settingRequiresTimeSeries(Map, Object> settings, Setting setting) { diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 9e62cf36d0cfc..791163d8707c9 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -699,13 +700,13 @@ public void testUpdateTimeSeriesTimeRange() { // smaller IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(endTime - randomLongBetween(1, 1000))) + () -> indexSettings.updateTimeSeriesEndTime(Optional.of(Instant.ofEpochMilli(endTime - randomLongBetween(1, 1000)))) ); assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than current value")); // success long newEndTime = endTime + randomLongBetween(1, 1000); - indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(newEndTime)); + indexSettings.updateTimeSeriesEndTime(Optional.of(Instant.ofEpochMilli(newEndTime))); assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); } From 40f2e82c5a5560f36c275f670884d05a1d206420 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 11 Nov 2021 22:24:28 +0800 Subject: [PATCH 18/24] fix failed tests --- .../common/settings/Setting.java | 14 ++---- .../elasticsearch/index/IndexSettings.java | 45 +++++++------------ .../DataStreamTimestampFieldMapper.java | 1 - .../index/mapper/DateFieldMapper.java | 2 +- .../index/IndexSettingsTests.java | 22 ++------- 5 files changed, 23 insertions(+), 61 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 2bef52ef2b8d7..57a6e0e3a4a19 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -43,7 +43,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BiFunction; @@ -1334,16 +1333,11 @@ public static Setting longSetting(String key, long defaultValue, long minV ); } - public static Setting> dateSetting( - String key, - Optional defaultValue, - Validator> validator, - Property... properties - ) { - return new Setting>( + public static Setting dateSetting(String key, Instant defaultValue, Validator validator, Property... properties) { + return new Setting<>( key, - defaultValue.map(Instant::toString).orElse(null), - s -> Optional.ofNullable(s).map(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER::parse).map(Instant::from), + defaultValue.toString(), + (s) -> Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(s)), validator, properties ); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index ed3070e2eab55..53f5fe0ab4cb5 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Booleans; @@ -31,7 +32,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; @@ -479,9 +479,9 @@ public static boolean isTimeSeriesModeEnabled() { /** * in time series mode, the start time of the index, timestamp must larger than start_time */ - public static final Setting> TIME_SERIES_START_TIME = Setting.dateSetting( + public static final Setting TIME_SERIES_START_TIME = Setting.dateSetting( "index.time_series.start_time", - Optional.empty(), + Instant.ofEpochMilli(0), v -> {}, Property.IndexScope, Property.Final @@ -490,28 +490,19 @@ public static boolean isTimeSeriesModeEnabled() { /** * in time series mode, the end time of the index, timestamp must smaller than start_time */ - public static final Setting> TIME_SERIES_END_TIME = Setting.dateSetting( + public static final Setting TIME_SERIES_END_TIME = Setting.dateSetting( "index.time_series.end_time", - Optional.empty(), + DateUtils.MAX_NANOSECOND_INSTANT, new Setting.Validator<>() { @Override - public void validate(Optional value) {} + public void validate(Instant value) {} @Override - public void validate(Optional value, Map, Object> settings) { + public void validate(Instant value, Map, Object> settings) { @SuppressWarnings("unchecked") - Optional startTime = (Optional) settings.get(TIME_SERIES_START_TIME); - if (value.isEmpty()) { - if (startTime.isPresent()) { - throw new IllegalArgumentException("index.time_series.start_time must be set with index.time_series.end_time"); - } - return; - } - if (startTime.isEmpty()) { - throw new IllegalArgumentException("index.time_series.start_time must be set with index.time_series.end_time"); - } - if (startTime.get().compareTo(value.get()) < 0) { - throw new IllegalArgumentException("index.time_series.start_time must be larger than index.time_series.end_time"); + Instant startTime = (Instant) settings.get(TIME_SERIES_START_TIME); + if (startTime.toEpochMilli() > value.toEpochMilli()) { + throw new IllegalArgumentException("index.time_series.end_time must be larger than index.time_series.start_time"); } } @@ -710,9 +701,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD; - timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings).orElse(null); - timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings).orElse(null); - + timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings); + timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings); this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings); this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings); this.queryStringAnalyzeWildcard = QUERY_STRING_ANALYZE_WILDCARD.get(nodeSettings); @@ -1349,17 +1339,12 @@ public Instant getTimeSeriesEndTime() { return timeSeriesEndTime; } - public void updateTimeSeriesEndTime(Optional endTime) { - if (endTime.isEmpty()) { - this.timeSeriesEndTime = null; - return; - } - Instant endInstant = endTime.get(); - if (this.timeSeriesEndTime.isAfter(endInstant)) { + public void updateTimeSeriesEndTime(Instant endTime) { + if (this.timeSeriesEndTime.isAfter(endTime)) { throw new IllegalArgumentException( "index.time_series.end_time must be larger than current value [" + this.timeSeriesEndTime + "]" ); } - this.timeSeriesEndTime = endInstant; + this.timeSeriesEndTime = endTime; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 056013ac0f8d7..dbadb9d1b9087 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -13,7 +13,6 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index b9d2bb034a9f0..7ce749ecbc024 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -211,7 +211,7 @@ public final void validateTimestamp(long value, DocumentParserContext context) { throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be larger than " + startTime); } - long endTime = convert(context.indexSettings().getTimeSeriesStartTime()); + long endTime = convert(context.indexSettings().getTimeSeriesEndTime()); if (value >= endTime) { throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be smaller than " + endTime); } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 791163d8707c9..660a1213575e2 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -26,7 +26,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -700,14 +699,14 @@ public void testUpdateTimeSeriesTimeRange() { // smaller IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> indexSettings.updateTimeSeriesEndTime(Optional.of(Instant.ofEpochMilli(endTime - randomLongBetween(1, 1000)))) + () -> indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(endTime - randomLongBetween(1, 1000))) ); assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than current value")); // success long newEndTime = endTime + randomLongBetween(1, 1000); - indexSettings.updateTimeSeriesEndTime(Optional.of(Instant.ofEpochMilli(newEndTime))); - assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); + indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(newEndTime)); + assertEquals(Instant.ofEpochMilli(newEndTime), indexSettings.getTimeSeriesEndTime()); } public void testTimeSeriesTimeBoundary() { @@ -724,19 +723,4 @@ public void testTimeSeriesTimeBoundary() { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time must be larger than index.time_series.start_time")); } - - public void testTimeSeriesTimeRangeModeNotSet() { - long time = System.currentTimeMillis(); - { - final Settings settings = Settings.builder().put(TIME_SERIES_START_TIME.getKey(), time).build(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TIME_SERIES_START_TIME.get(settings)); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.start_time need to be used for time_series mode")); - } - - { - final Settings settings = Settings.builder().put(TIME_SERIES_END_TIME.getKey(), time).build(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> TIME_SERIES_END_TIME.get(settings)); - assertThat(e.getMessage(), Matchers.containsString("index.time_series.end_time need to be used for time_series mode")); - } - } } From 582588fffab8cff275d0be16fef2eb6da0a25517 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 11 Nov 2021 22:39:48 +0800 Subject: [PATCH 19/24] fix failed tests --- .../resources/rest-api-spec/test/tsdb/10_settings.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index d03f62b89e70c..5b3dc0ad340d4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -194,7 +194,7 @@ set start_time and end_time: end_time: 1632625793000 - do: - catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/ + catch: /index.time_series.end_time must be larger than current value \[2021-09-26T03:09:53Z\]/ indices.put_settings: index: test_index body: @@ -212,7 +212,7 @@ set start_time and end_time without timeseries mode: version: " - 8.0.99" reason: introduced in 8.1.0 - do: - catch: /index.time_series.start_time need to be used for time_series mode/ + catch: /\[index.time_series.start_time\] requires \[index.mode=time_series\]/ indices.create: index: test_index body: @@ -222,7 +222,7 @@ set start_time and end_time without timeseries mode: start_time: 1632625782000 - do: - catch: /index.time_series.end_time need to be used for time_series mode/ + catch: /\[index.time_series.end_time\] requires \[index.mode=time_series\]/ indices.create: index: test_index body: From bfe25c560f690d3c5c09c8ba977a5621ced98b4d Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 11 Nov 2021 23:29:41 +0800 Subject: [PATCH 20/24] improve --- .../elasticsearch/index/IndexSettings.java | 17 +++++----- .../DataStreamTimestampFieldMapper.java | 25 +++++++++++++++ .../index/mapper/DateFieldMapper.java | 31 ++----------------- .../index/IndexSettingsTests.java | 2 +- .../index/TimeSeriesModeTests.java | 6 ++-- 5 files changed, 41 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 53f5fe0ab4cb5..6fdcd184e9264 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -554,11 +554,11 @@ public Iterator> settings() { /** * Start time of the time_series index. */ - private final Instant timeSeriesStartTime; + private final long timeSeriesStartTime; /** * End time of the time_series index. */ - private volatile Instant timeSeriesEndTime; + private volatile long timeSeriesEndTime; // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock private volatile Settings settings; @@ -701,8 +701,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); mode = isTimeSeriesModeEnabled() ? scopedSettings.get(MODE) : IndexMode.STANDARD; - timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings); - timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings); + timeSeriesStartTime = TIME_SERIES_START_TIME.get(settings).toEpochMilli(); + timeSeriesEndTime = TIME_SERIES_END_TIME.get(settings).toEpochMilli(); this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings); this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings); this.queryStringAnalyzeWildcard = QUERY_STRING_ANALYZE_WILDCARD.get(nodeSettings); @@ -1331,16 +1331,17 @@ private void setMappingDimensionFieldsLimit(long value) { this.mappingDimensionFieldsLimit = value; } - public Instant getTimeSeriesStartTime() { + public long getTimeSeriesStartTime() { return timeSeriesStartTime; } - public Instant getTimeSeriesEndTime() { + public long getTimeSeriesEndTime() { return timeSeriesEndTime; } - public void updateTimeSeriesEndTime(Instant endTime) { - if (this.timeSeriesEndTime.isAfter(endTime)) { + public void updateTimeSeriesEndTime(Instant endTimeInstant) { + long endTime = endTimeInstant.toEpochMilli(); + if (this.timeSeriesEndTime > endTime) { throw new IllegalArgumentException( "index.time_series.end_time must be larger than current value [" + this.timeSeriesEndTime + "]" ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index dbadb9d1b9087..ea1f6c85a9c17 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.core.TimeValue.NSEC_PER_MSEC; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; /** @@ -210,6 +212,29 @@ public void postParse(DocumentParserContext context) throws IOException { if (numberOfValues > 1) { throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] encountered multiple values"); } + + validateTimestamp(fields[0], context); + } + + private final void validateTimestamp(IndexableField field, DocumentParserContext context) { + if (context.indexSettings().getMode() == null || context.indexSettings().getMode() != IndexMode.TIME_SERIES) { + return ; + } + + long value = field.numericValue().longValue(); + if (context.mappingLookup().getMapper(DEFAULT_PATH).typeName().equals(DateFieldMapper.DATE_NANOS_CONTENT_TYPE)) { + value /= NSEC_PER_MSEC; + } + + long startTime = context.indexSettings().getTimeSeriesStartTime(); + if (value < startTime) { + throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be larger than " + startTime); + } + + long endTime = context.indexSettings().getTimeSeriesEndTime(); + if (value >= endTime) { + throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be smaller than " + endTime); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 7ce749ecbc024..d8cd8c781a8b9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -201,22 +201,6 @@ NumericType numericType() { */ public abstract long roundUpToMillis(long value); - public final void validateTimestamp(long value, DocumentParserContext context) { - if (context.indexSettings().getTimeSeriesStartTime() == null) { - assert context.indexSettings().getTimeSeriesEndTime() == null; - return; - } - long startTime = convert(context.indexSettings().getTimeSeriesStartTime()); - if (value < startTime) { - throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be larger than " + startTime); - } - - long endTime = convert(context.indexSettings().getTimeSeriesEndTime()); - if (value >= endTime) { - throw new IllegalArgumentException("time series index @timestamp value [" + value + "] must be smaller than " + endTime); - } - } - public static Resolution ofOrdinal(int ord) { for (Resolution resolution : values()) { if (ord == resolution.ordinal()) { @@ -260,7 +244,6 @@ public static class Builder extends FieldMapper.Builder { private final Resolution resolution; private final Version indexCreatedVersion; private final ScriptCompiler scriptCompiler; - private final Validate validate; public Builder( String name, @@ -287,7 +270,6 @@ public Builder( this.format.setValue(dateFormatter.pattern()); this.locale.setValue(dateFormatter.locale()); } - this.validate = name.equals(DataStreamTimestampFieldMapper.DEFAULT_PATH) ? resolution::validateTimestamp : (v, c) -> {}; } private DateFormatter buildFormatter() { @@ -361,8 +343,7 @@ public DateFieldMapper build(MapperBuilderContext context) { copyTo.build(), nullTimestamp, resolution, - this, - validate + this ); } } @@ -743,7 +724,6 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { private final Script script; private final ScriptCompiler scriptCompiler; private final FieldValues scriptValues; - private final Validate validate; private DateFieldMapper( String simpleName, @@ -752,8 +732,7 @@ private DateFieldMapper( CopyTo copyTo, Long nullValue, Resolution resolution, - Builder builder, - Validate validate + Builder builder ) { super(simpleName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.get()); this.store = builder.store.getValue(); @@ -770,7 +749,6 @@ private DateFieldMapper( this.script = builder.script.get(); this.scriptCompiler = builder.scriptCompiler; this.scriptValues = builder.scriptValues(); - this.validate = validate; } @Override @@ -810,7 +788,6 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } } } - validate.validate(timestamp, context); indexValue(context, timestamp); } @@ -846,8 +823,4 @@ public boolean getIgnoreMalformed() { public Long getNullValue() { return nullValue; } - - private interface Validate { - void validate(long value, DocumentParserContext context); - } } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 660a1213575e2..069412b998e69 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -706,7 +706,7 @@ public void testUpdateTimeSeriesTimeRange() { // success long newEndTime = endTime + randomLongBetween(1, 1000); indexSettings.updateTimeSeriesEndTime(Instant.ofEpochMilli(newEndTime)); - assertEquals(Instant.ofEpochMilli(newEndTime), indexSettings.getTimeSeriesEndTime()); + assertEquals(newEndTime, indexSettings.getTimeSeriesEndTime()); } public void testTimeSeriesTimeBoundary() { diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index b47f8f25e5132..ce93dbe204b78 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -178,8 +178,10 @@ public void testEnableTimestampRange() throws IOException { .put(IndexSettings.MODE.getKey(), "time_series") .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") .build(); - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); + DocumentMapper mapper = createMapperService( + s, + mapping(b -> b.startObject("@timestamp").field("type", randomBoolean() ? "date" : "date_nanos").endObject()) + ).documentMapper(); mapper.parse( new SourceToParse( "test", From 6350ee1827e91c145978647bdfcc3b3602d6953b Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 11 Nov 2021 23:34:38 +0800 Subject: [PATCH 21/24] checkStyle --- .../index/mapper/DataStreamTimestampFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index ea1f6c85a9c17..1cf16b0b728ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -216,7 +216,7 @@ public void postParse(DocumentParserContext context) throws IOException { validateTimestamp(fields[0], context); } - private final void validateTimestamp(IndexableField field, DocumentParserContext context) { + private void validateTimestamp(IndexableField field, DocumentParserContext context) { if (context.indexSettings().getMode() == null || context.indexSettings().getMode() != IndexMode.TIME_SERIES) { return ; } From a5a116f26b05b2bc70f4fbe5998fff4293f98ac5 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 11 Nov 2021 23:36:33 +0800 Subject: [PATCH 22/24] spotless --- .../index/mapper/DataStreamTimestampFieldMapper.java | 2 +- .../elasticsearch/index/mapper/DateFieldMapper.java | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 1cf16b0b728ee..9ff6abeffd143 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -218,7 +218,7 @@ public void postParse(DocumentParserContext context) throws IOException { private void validateTimestamp(IndexableField field, DocumentParserContext context) { if (context.indexSettings().getMode() == null || context.indexSettings().getMode() != IndexMode.TIME_SERIES) { - return ; + return; } long value = field.numericValue().longValue(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index d8cd8c781a8b9..4e07b9bcfb1b1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -336,15 +336,7 @@ public DateFieldMapper build(MapperBuilderContext context) { ); Long nullTimestamp = parseNullValue(ft); - return new DateFieldMapper( - name, - ft, - multiFieldsBuilder.build(this, context), - copyTo.build(), - nullTimestamp, - resolution, - this - ); + return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); } } From 7236ee115686fda8baa4c1655c3831eb9fa9ce13 Mon Sep 17 00:00:00 2001 From: weizijun Date: Thu, 11 Nov 2021 23:47:58 +0800 Subject: [PATCH 23/24] fixex tsdb settings tests --- .../resources/rest-api-spec/test/tsdb/10_settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index 5b3dc0ad340d4..4df1579cd4597 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -194,7 +194,7 @@ set start_time and end_time: end_time: 1632625793000 - do: - catch: /index.time_series.end_time must be larger than current value \[2021-09-26T03:09:53Z\]/ + catch: /index.time_series.end_time must be larger than current value \[1632625793000\]/ indices.put_settings: index: test_index body: From 9e92d098bfd5162abf91df9d8be74da26e3d35d7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 11 Nov 2021 11:57:39 -0500 Subject: [PATCH 24/24] Fixup after merge --- .../java/org/elasticsearch/index/IndexMode.java | 1 + .../elasticsearch/index/TimeSeriesModeTests.java | 15 +++++---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index e6103eef98372..7c672d65632c3 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -73,6 +73,7 @@ void validateWithOtherSettings(Map, Object> settings) { } } settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH); + // TODO make start and stop time required } private void settingRequiresTimeSeries(Map, Object> settings, Setting setting) { diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index ce93dbe204b78..d16c1d273207c 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -157,12 +158,7 @@ public void testWithoutTimestamp() throws IOException { MapperParsingException e = expectThrows( MapperParsingException.class, () -> mapper.parse( - new SourceToParse( - "test", - "1", - BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()), - XContentType.JSON - ) + new SourceToParse("1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()), XContentType.JSON) ) ); assertThat(e.getRootCause().getMessage(), containsString("data stream timestamp field [@timestamp] is missing")); @@ -182,9 +178,8 @@ public void testEnableTimestampRange() throws IOException { s, mapping(b -> b.startObject("@timestamp").field("type", randomBoolean() ? "date" : "date_nanos").endObject()) ).documentMapper(); - mapper.parse( + ParsedDocument doc = mapper.parse( new SourceToParse( - "test", "1", BytesReference.bytes( XContentFactory.jsonBuilder().startObject().field("@timestamp", randomLongBetween(startTime, endTime)).endObject() @@ -192,6 +187,8 @@ public void testEnableTimestampRange() throws IOException { XContentType.JSON ) ); + // Look, mah, no failure. + assertNotNull(doc.rootDoc().getNumericValue("@timestamp")); } public void testBadStartTime() throws IOException { @@ -211,7 +208,6 @@ public void testBadStartTime() throws IOException { MapperParsingException.class, () -> mapper.parse( new SourceToParse( - "test", "1", BytesReference.bytes( XContentFactory.jsonBuilder() @@ -243,7 +239,6 @@ public void testBadEndTime() throws IOException { MapperParsingException.class, () -> mapper.parse( new SourceToParse( - "test", "1", BytesReference.bytes( XContentFactory.jsonBuilder().startObject().field("@timestamp", endTime + randomLongBetween(0, 3)).endObject()