Skip to content

Commit 61cd70b

Browse files
martijnvgmartijnvg
authored
TSDB: Truncate timestamps for routing purposes only (#84699)
Truncate timestamp to second precision for resolving data stream to backing index only. Also add support for parsing timestamps in date nanos format. Closes #83517
1 parent 22824e4 commit 61cd70b

File tree

7 files changed

+443
-35
lines changed

7 files changed

+443
-35
lines changed

‎modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java‎

Lines changed: 91 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -159,34 +159,34 @@ public class TsdbDataStreamRestIT extends ESRestTestCase {
159159
}
160160
""";
161161

162+
private static final String BULK =
163+
"""
164+
{"create": {}}
165+
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}
166+
{"create": {}}
167+
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}
168+
{"create": {}}
169+
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}
170+
{"create": {}}
171+
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}
172+
{"create": {}}
173+
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}
174+
{"create": {}}
175+
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}
176+
{"create": {}}
177+
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}
178+
{"create": {}}
179+
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}
180+
""";
181+
162182
public void testTsdbDataStreams() throws Exception {
163183
// Create a template
164184
var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
165185
putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE);
166186
assertOK(client().performRequest(putComposableIndexTemplateRequest));
167187

168188
var bulkRequest = new Request("POST", "/k8s/_bulk");
169-
bulkRequest.setJsonEntity(
170-
"""
171-
{"create": {}}
172-
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}
173-
{"create": {}}
174-
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}
175-
{"create": {}}
176-
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}
177-
{"create": {}}
178-
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}
179-
{"create": {}}
180-
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}
181-
{"create": {}}
182-
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}
183-
{"create": {}}
184-
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}
185-
{"create": {}}
186-
{"@timestamp": "$now", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}
187-
"""
188-
.replace("$now", formatInstant(Instant.now()))
189-
);
189+
bulkRequest.setJsonEntity(BULK.replace("$now", formatInstant(Instant.now())));
190190
bulkRequest.addParameter("refresh", "true");
191191
assertOK(client().performRequest(bulkRequest));
192192

@@ -245,6 +245,72 @@ public void testTsdbDataStreams() throws Exception {
245245
assertThat(entityAsMap(response).get("_index"), equalTo(secondBackingIndex));
246246
}
247247

248+
public void testTsdbDataStreamsNanos() throws Exception {
249+
// Create a template
250+
var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
251+
putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE.replace("date", "date_nanos"));
252+
assertOK(client().performRequest(putComposableIndexTemplateRequest));
253+
254+
var bulkRequest = new Request("POST", "/k8s/_bulk");
255+
bulkRequest.setJsonEntity(BULK.replace("$now", formatInstantNanos(Instant.now())));
256+
bulkRequest.addParameter("refresh", "true");
257+
assertOK(client().performRequest(bulkRequest));
258+
259+
var getDataStreamsRequest = new Request("GET", "/_data_stream");
260+
var response = client().performRequest(getDataStreamsRequest);
261+
assertOK(response);
262+
var dataStreams = entityAsMap(response);
263+
assertThat(ObjectPath.evaluate(dataStreams, "data_streams"), hasSize(1));
264+
assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.name"), equalTo("k8s"));
265+
assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.generation"), equalTo(1));
266+
assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.template"), equalTo("1"));
267+
assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.indices"), hasSize(1));
268+
String firstBackingIndex = ObjectPath.evaluate(dataStreams, "data_streams.0.indices.0.index_name");
269+
assertThat(firstBackingIndex, backingIndexEqualTo("k8s", 1));
270+
271+
var indices = getIndex(firstBackingIndex);
272+
var escapedBackingIndex = firstBackingIndex.replace(".", "\\.");
273+
assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".data_stream"), equalTo("k8s"));
274+
assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.mode"), equalTo("time_series"));
275+
String startTimeFirstBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.start_time");
276+
assertThat(startTimeFirstBackingIndex, notNullValue());
277+
String endTimeFirstBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.end_time");
278+
assertThat(endTimeFirstBackingIndex, notNullValue());
279+
280+
var rolloverRequest = new Request("POST", "/k8s/_rollover");
281+
assertOK(client().performRequest(rolloverRequest));
282+
283+
response = client().performRequest(getDataStreamsRequest);
284+
assertOK(response);
285+
dataStreams = entityAsMap(response);
286+
assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.name"), equalTo("k8s"));
287+
assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.generation"), equalTo(2));
288+
String secondBackingIndex = ObjectPath.evaluate(dataStreams, "data_streams.0.indices.1.index_name");
289+
assertThat(secondBackingIndex, backingIndexEqualTo("k8s", 2));
290+
291+
indices = getIndex(secondBackingIndex);
292+
escapedBackingIndex = secondBackingIndex.replace(".", "\\.");
293+
assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".data_stream"), equalTo("k8s"));
294+
String startTimeSecondBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.start_time");
295+
assertThat(startTimeSecondBackingIndex, equalTo(endTimeFirstBackingIndex));
296+
String endTimeSecondBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.end_time");
297+
assertThat(endTimeSecondBackingIndex, notNullValue());
298+
299+
var indexRequest = new Request("POST", "/k8s/_doc");
300+
Instant time = parseInstant(startTimeFirstBackingIndex);
301+
indexRequest.setJsonEntity(DOC.replace("$time", formatInstantNanos(time)));
302+
response = client().performRequest(indexRequest);
303+
assertOK(response);
304+
assertThat(entityAsMap(response).get("_index"), equalTo(firstBackingIndex));
305+
306+
indexRequest = new Request("POST", "/k8s/_doc");
307+
time = parseInstant(endTimeSecondBackingIndex).minusMillis(1);
308+
indexRequest.setJsonEntity(DOC.replace("$time", formatInstantNanos(time)));
309+
response = client().performRequest(indexRequest);
310+
assertOK(response);
311+
assertThat(entityAsMap(response).get("_index"), equalTo(secondBackingIndex));
312+
}
313+
248314
public void testSimulateTsdbDataStreamTemplate() throws Exception {
249315
var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");
250316
putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE);
@@ -429,6 +495,10 @@ static String formatInstant(Instant instant) {
429495
return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant);
430496
}
431497

498+
static String formatInstantNanos(Instant instant) {
499+
return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME_NANOS.getName()).format(instant);
500+
}
501+
432502
static Instant parseInstant(String input) {
433503
return Instant.from(DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).parse(input));
434504
}

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.index.mapper.DateFieldMapper;
2020

2121
import java.time.Instant;
22+
import java.time.temporal.ChronoUnit;
2223
import java.util.Locale;
2324

2425
public class DataStreamIndexSettingsProvider implements IndexSettingProvider {
@@ -59,8 +60,8 @@ public Settings getAdditionalIndexSettings(
5960
final Instant start;
6061
final Instant end;
6162
if (dataStream == null || migrating) {
62-
start = resolvedAt.minusMillis(lookAheadTime.getMillis());
63-
end = resolvedAt.plusMillis(lookAheadTime.getMillis());
63+
start = resolvedAt.minusMillis(lookAheadTime.getMillis()).truncatedTo(ChronoUnit.SECONDS);
64+
end = resolvedAt.plusMillis(lookAheadTime.getMillis()).truncatedTo(ChronoUnit.SECONDS);
6465
} else {
6566
IndexMetadata currentLatestBackingIndex = metadata.index(dataStream.getWriteIndex());
6667
if (currentLatestBackingIndex.getSettings().hasValue(IndexSettings.TIME_SERIES_END_TIME.getKey()) == false) {
@@ -75,9 +76,9 @@ public Settings getAdditionalIndexSettings(
7576
}
7677
start = IndexSettings.TIME_SERIES_END_TIME.get(currentLatestBackingIndex.getSettings());
7778
if (start.isAfter(resolvedAt)) {
78-
end = start.plusMillis(lookAheadTime.getMillis());
79+
end = start.plusMillis(lookAheadTime.getMillis()).truncatedTo(ChronoUnit.SECONDS);
7980
} else {
80-
end = resolvedAt.plusMillis(lookAheadTime.getMillis());
81+
end = resolvedAt.plusMillis(lookAheadTime.getMillis()).truncatedTo(ChronoUnit.SECONDS);
8182
}
8283
}
8384
assert start.isBefore(end) : "data stream backing index's start time is not before end time";

0 commit comments

Comments
 (0)