Skip to content

Commit 986b0a8

Browse files
weizijunweizijun
committed
add timestamp meta field to time_series index
1 parent c4a4b07 commit 986b0a8

File tree

10 files changed

+167
-90
lines changed

10 files changed

+167
-90
lines changed

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ date:
1212
settings:
1313
index:
1414
mode: time_series
15+
routing_path: [metricset]
1516
number_of_replicas: 0
1617
number_of_shards: 2
1718
mappings:
@@ -20,12 +21,13 @@ date:
2021
type: date
2122
metricset:
2223
type: keyword
23-
dimension: true
24+
time_series_dimension: true
2425

2526
- do:
2627
indices.get_mapping:
2728
index: test
2829
- match: { "[email protected]": date }
30+
- match: { 'test.mappings._data_stream_timestamp.enabled': true }
2931

3032
- do:
3133
bulk:
@@ -56,6 +58,7 @@ date_nanos:
5658
settings:
5759
index:
5860
mode: time_series
61+
routing_path: [metricset]
5962
number_of_replicas: 0
6063
number_of_shards: 2
6164
mappings:
@@ -64,12 +67,13 @@ date_nanos:
6467
type: date_nanos
6568
metricset:
6669
type: keyword
67-
dimension: true
70+
time_series_dimension: true
6871

6972
- do:
7073
indices.get_mapping:
7174
index: test
7275
- match: { "[email protected]": date_nanos }
76+
- match: { 'test.mappings._data_stream_timestamp.enabled': true }
7377

7478
- do:
7579
bulk:
@@ -100,18 +104,20 @@ automatically add with date:
100104
settings:
101105
index:
102106
mode: time_series
107+
routing_path: [metricset]
103108
number_of_replicas: 0
104109
number_of_shards: 2
105110
mappings:
106111
properties:
107112
metricset:
108113
type: keyword
109-
dimension: true
114+
time_series_dimension: true
110115

111116
- do:
112117
indices.get_mapping:
113118
index: test
114119
- match: { 'test.mappings.properties.@timestamp': { "type": date } }
120+
- match: { 'test.mappings._data_stream_timestamp.enabled': true }
115121

116122
- do:
117123
bulk:
@@ -136,16 +142,38 @@ reject @timestamp with wrong type:
136142
reason: introduced in 8.0.0 to be backported to 7.16.0
137143

138144
- do:
139-
catch: /data stream timestamp field \[@timestamp\] is of type keyword, but \[date,date_nanos\] is expected/
145+
catch: /data stream timestamp field \[@timestamp\] is of type \[keyword\], but \[date,date_nanos\] is expected/
140146
indices.create:
141147
index: test
142148
body:
143149
settings:
144150
index:
145151
mode: time_series
152+
routing_path: [metricset]
146153
number_of_replicas: 0
147154
number_of_shards: 2
148155
mappings:
149156
properties:
150157
"@timestamp":
151158
type: keyword
159+
160+
---
161+
reject timestamp meta field with wrong type:
162+
- skip:
163+
version: " - 7.99.99"
164+
reason: introduced in 8.0.0 to be backported to 7.16.0
165+
166+
- do:
167+
catch: /.* time series index \[_data_stream_timestamp\] meta field must be enabled/
168+
indices.create:
169+
index: test
170+
body:
171+
settings:
172+
index:
173+
mode: time_series
174+
routing_path: [metricset]
175+
number_of_replicas: 0
176+
number_of_shards: 2
177+
mappings:
178+
_data_stream_timestamp:
179+
enabled: false

‎server/src/main/java/org/elasticsearch/index/IndexMode.java‎

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,17 @@
1111
import org.elasticsearch.cluster.metadata.IndexMetadata;
1212
import org.elasticsearch.common.settings.Setting;
1313
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.common.util.Maps;
1415
import org.elasticsearch.core.Nullable;
1516
import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
1617
import org.elasticsearch.index.mapper.DateFieldMapper;
1718
import org.elasticsearch.index.mapper.Mapper;
1819
import org.elasticsearch.index.mapper.MappingLookup;
1920
import org.elasticsearch.index.mapper.MappingParserContext;
20-
import org.elasticsearch.index.mapper.MetadataFieldMapper;
2121
import org.elasticsearch.index.mapper.RootObjectMapper;
2222
import org.elasticsearch.index.mapper.RoutingFieldMapper;
2323

24+
import java.util.HashMap;
2425
import java.util.List;
2526
import java.util.Map;
2627
import java.util.Objects;
@@ -52,11 +53,7 @@ void validateWithOtherSettings(Map<Setting<?>, Object> settings) {
5253
public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) {}
5354

5455
@Override
55-
public void completeMappings(
56-
MappingParserContext context,
57-
Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers,
58-
RootObjectMapper.Builder builder
59-
) {}
56+
public void completeMappings(MappingParserContext context, Map<String, Object> mapping, RootObjectMapper.Builder builder) {}
6057
},
6158
TIME_SERIES {
6259
@Override
@@ -102,16 +99,11 @@ private String tsdbMode() {
10299
}
103100

104101
@Override
105-
public void completeMappings(
106-
MappingParserContext context,
107-
Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers,
108-
RootObjectMapper.Builder builder
109-
) {
110-
DataStreamTimestampFieldMapper timestampFieldMapper = (DataStreamTimestampFieldMapper) metadataMappers.get(
111-
DataStreamTimestampFieldMapper.class
112-
);
113-
if (timestampFieldMapper == null || false == timestampFieldMapper.isEnabled()) {
114-
metadataMappers.put(DataStreamTimestampFieldMapper.class, DataStreamTimestampFieldMapper.ENABLED_INSTANCE);
102+
public void completeMappings(MappingParserContext context, Map<String, Object> mapping, RootObjectMapper.Builder builder) {
103+
if (false == mapping.containsKey(DataStreamTimestampFieldMapper.NAME)) {
104+
mapping.put(DataStreamTimestampFieldMapper.NAME, new HashMap<>(Map.of("enabled", true)));
105+
} else {
106+
validateTimeStampField(mapping.get(DataStreamTimestampFieldMapper.NAME));
115107
}
116108

117109
Optional<Mapper.Builder> timestamp = builder.getBuilder(DataStreamTimestampFieldMapper.DEFAULT_PATH);
@@ -128,6 +120,23 @@ public void completeMappings(
128120
);
129121
}
130122
}
123+
124+
private void validateTimeStampField(Object timestampFieldValue) {
125+
if (false == (timestampFieldValue instanceof Map)) {
126+
throw new IllegalArgumentException(
127+
"time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field format error"
128+
);
129+
}
130+
131+
@SuppressWarnings("unchecked")
132+
Map<String, Object> timeStampFieldValueMap = (Map<String, Object>) timestampFieldValue;
133+
if (false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", true))
134+
&& false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", "true"))) {
135+
throw new IllegalArgumentException(
136+
"time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field must be enabled"
137+
);
138+
}
139+
}
131140
};
132141

133142
private static final List<Setting<?>> TIME_SERIES_UNSUPPORTED = List.of(
@@ -159,9 +168,5 @@ public void completeMappings(
159168
/**
160169
* Validate and/or modify the mappings after after they've been parsed.
161170
*/
162-
public abstract void completeMappings(
163-
MappingParserContext context,
164-
Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers,
165-
RootObjectMapper.Builder builder
166-
);
171+
public abstract void completeMappings(MappingParserContext context, Map<String, Object> mapping, RootObjectMapper.Builder builder);
167172
}

‎server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -94,11 +94,9 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar
9494

9595
private Mapping parse(String type, Map<String, Object> mapping) throws MapperParsingException {
9696
MappingParserContext parserContext = parserContextSupplier.get();
97-
98-
// parserContext.getIndexSettings().getMode().completeMappings(parserContext, mapping);
99-
10097
RootObjectMapper.Builder rootObjectMapperBuilder
10198
= rootObjectTypeParser.parse(type, mapping, parserContext);
99+
parserContext.getIndexSettings().getMode().completeMappings(parserContext, mapping, rootObjectMapperBuilder);
102100

103101
Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get();
104102
Map<String, Object> meta = null;
@@ -146,8 +144,6 @@ private Mapping parse(String type, Map<String, Object> mapping) throws MapperPar
146144
}
147145
checkNoRemainingFields(mapping, "Root mapping definition has unsupported parameters: ");
148146

149-
parserContext.getIndexSettings().getMode().completeMappings(parserContext, metadataMappers, rootObjectMapperBuilder);
150-
151147
return new Mapping(
152148
rootObjectMapperBuilder.build(MapperBuilderContext.ROOT),
153149
metadataMappers.values().toArray(new MetadataFieldMapper[0]),

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,9 @@
1616
import org.elasticsearch.index.Index;
1717
import org.elasticsearch.index.mapper.MapperService;
1818
import org.elasticsearch.index.mapper.MapperServiceTestCase;
19-
import org.elasticsearch.plugins.Plugin;
2019

2120
import java.io.IOException;
2221
import java.util.Arrays;
23-
import java.util.Collection;
2422
import java.util.List;
2523
import java.util.stream.Collectors;
2624

@@ -352,9 +350,4 @@ private MapperService getMapperService(IndexMetadata im) {
352350
throw new IllegalStateException(e);
353351
}
354352
}
355-
356-
@Override
357-
protected Collection<? extends Plugin> getPlugins() {
358-
return List.of(new MetadataIndexTemplateServiceTests.DummyPlugin());
359-
}
360353
}

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java‎

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
package org.elasticsearch.cluster.metadata;
1010

1111
import com.fasterxml.jackson.core.JsonParseException;
12+
1213
import org.elasticsearch.ResourceNotFoundException;
1314
import org.elasticsearch.Version;
1415
import org.elasticsearch.action.ActionListener;
@@ -23,27 +24,22 @@
2324
import org.elasticsearch.common.settings.IndexScopedSettings;
2425
import org.elasticsearch.common.settings.Settings;
2526
import org.elasticsearch.core.TimeValue;
26-
import org.elasticsearch.test.ClusterServiceUtils;
27-
import org.elasticsearch.threadpool.ThreadPool;
28-
import org.elasticsearch.xcontent.NamedXContentRegistry;
29-
import org.elasticsearch.xcontent.XContentFactory;
3027
import org.elasticsearch.env.Environment;
3128
import org.elasticsearch.index.Index;
32-
import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
3329
import org.elasticsearch.index.mapper.MapperParsingException;
3430
import org.elasticsearch.index.mapper.MapperService;
35-
import org.elasticsearch.index.mapper.MetadataFieldMapper;
3631
import org.elasticsearch.indices.EmptySystemIndices;
3732
import org.elasticsearch.indices.IndexTemplateMissingException;
3833
import org.elasticsearch.indices.IndicesService;
3934
import org.elasticsearch.indices.InvalidIndexTemplateException;
40-
import org.elasticsearch.plugins.MapperPlugin;
41-
import org.elasticsearch.plugins.Plugin;
35+
import org.elasticsearch.test.ClusterServiceUtils;
4236
import org.elasticsearch.test.ESSingleNodeTestCase;
37+
import org.elasticsearch.threadpool.ThreadPool;
38+
import org.elasticsearch.xcontent.NamedXContentRegistry;
39+
import org.elasticsearch.xcontent.XContentFactory;
4340

4441
import java.util.ArrayList;
4542
import java.util.Arrays;
46-
import java.util.Collection;
4743
import java.util.Collections;
4844
import java.util.HashMap;
4945
import java.util.HashSet;
@@ -78,11 +74,6 @@
7874

7975
public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
8076

81-
@Override
82-
protected Collection<Class<? extends Plugin>> getPlugins() {
83-
return List.of(DummyPlugin.class);
84-
}
85-
8677
public void testLegacyNoopUpdate() {
8778
ClusterState state = ClusterState.EMPTY_STATE;
8879
PutRequest pr = new PutRequest("api", "id");
@@ -1595,15 +1586,4 @@ clusterService, createIndexService, new AliasValidator(), indicesService,
15951586
public static void assertTemplatesEqual(ComposableIndexTemplate actual, ComposableIndexTemplate expected) {
15961587
assertTrue(Objects.equals(actual, expected));
15971588
}
1598-
1599-
// Composable index template with data_stream definition need _timestamp meta field mapper,
1600-
// this is a dummy impl, so that tests don't fail with the fact that the _timestamp field can't be found.
1601-
// (tests using this dummy impl doesn't test the _timestamp validation, but need it to tests other functionality)
1602-
public static class DummyPlugin extends Plugin implements MapperPlugin {
1603-
1604-
@Override
1605-
public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
1606-
return Map.of(DataStreamTimestampFieldMapper.NAME, DataStreamTimestampFieldMapper.PARSER);
1607-
}
1608-
}
16091589
}

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,8 @@
1818
import org.elasticsearch.index.mapper.MapperService;
1919
import org.elasticsearch.index.mapper.MapperServiceTestCase;
2020
import org.elasticsearch.indices.EmptySystemIndices;
21-
import org.elasticsearch.plugins.Plugin;
2221

2322
import java.io.IOException;
24-
import java.util.Collection;
2523
import java.util.List;
2624
import java.util.Set;
2725
import java.util.stream.Collectors;
@@ -326,9 +324,4 @@ private MetadataCreateIndexService getMetadataCreateIndexService() {
326324
when(service.getSystemIndices()).thenReturn(EmptySystemIndices.INSTANCE);
327325
return service;
328326
}
329-
330-
@Override
331-
protected Collection<? extends Plugin> getPlugins() {
332-
return List.of(new MetadataIndexTemplateServiceTests.DummyPlugin());
333-
}
334327
}

0 commit comments

Comments
 (0)