Skip to content

Commit ff6e589

Browse files
pgomulkapgomulka
authored
Deprecate resolution loss on date field (#78921) backport(#79355)
When storing nanoseconds on a date field the nanosecond part is lost as it cannot be stored. The date_nanos field should be used instead. This commit emits a deprecation warning to notify users about this. closes #37962 backports #78921
1 parent b7968e6 commit ff6e589

File tree

13 files changed

+101
-34
lines changed

13 files changed

+101
-34
lines changed

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ public void testDateNanosFormatUpgrade() throws IOException {
219219
Request index = new Request("POST", "/" + indexName + "/_doc/");
220220
XContentBuilder doc = XContentBuilder.builder(XContentType.JSON.xContent())
221221
.startObject()
222-
.field("date", "2015-01-01T12:10:30.123456789Z")
222+
.field("date", "2015-01-01T12:10:30.123Z")
223223
.field("date_nanos", "2015-01-01T12:10:30.123456789Z")
224224
.endObject();
225225
index.addParameter("refresh", "true");

‎qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
index: timetest
1010
body:
1111
mappings:
12-
"properties": { "my_time": {"type": "date", "format": "strict_date_optional_time_nanos"}}
12+
"properties": { "my_time": {"type": "date_nanos", "format": "strict_date_optional_time_nanos"}}
1313

1414
- do:
1515
ingest.put_pipeline:

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/49_range_timezone_bug.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ setup:
88
mappings:
99
properties:
1010
mydate:
11-
type: date
11+
type: date_nanos
1212
format: "uuuu-MM-dd'T'HH:mm:ss.SSSSSSSSSZZZZZ"
1313

1414
- do:

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 43 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ public final class DateFieldMapper extends FieldMapper {
7575
public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis");
7676
public static final DateFormatter DEFAULT_DATE_TIME_NANOS_FORMATTER =
7777
DateFormatter.forPattern("strict_date_optional_time_nanos||epoch_millis");
78+
private final String indexName;
7879

7980
public enum Resolution {
8081
MILLISECONDS(CONTENT_TYPE, NumericType.DATE) {
@@ -234,6 +235,7 @@ public static class Builder extends FieldMapper.Builder {
234235
private final Parameter<String> nullValue
235236
= Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null).acceptsNull();
236237
private final Parameter<Boolean> ignoreMalformed;
238+
private String indexName;
237239

238240
private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
239241
private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
@@ -244,13 +246,14 @@ public static class Builder extends FieldMapper.Builder {
244246

245247
public Builder(String name, Resolution resolution, DateFormatter dateFormatter,
246248
ScriptCompiler scriptCompiler,
247-
boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
249+
boolean ignoreMalformedByDefault, Version indexCreatedVersion, String indexName) {
248250
super(name);
249251
this.resolution = resolution;
250252
this.indexCreatedVersion = indexCreatedVersion;
251253
this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
252254
this.ignoreMalformed
253255
= Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
256+
this.indexName = indexName;
254257

255258
DateFormatter defaultFormat = resolution == Resolution.NANOSECONDS && indexCreatedVersion.onOrAfter(Version.V_7_0_0) ?
256259
DEFAULT_DATE_TIME_NANOS_FORMATTER : DEFAULT_DATE_TIME_FORMATTER;
@@ -292,12 +295,13 @@ protected List<Parameter<?>> getParameters() {
292295
return Arrays.asList(index, docValues, store, format, locale, nullValue, ignoreMalformed, script, onScriptError, boost, meta);
293296
}
294297

295-
private Long parseNullValue(DateFieldType fieldType) {
298+
private Long parseNullValue(DateFieldType fieldType, String indexName) {
296299
if (nullValue.getValue() == null) {
297300
return null;
298301
}
299302
try {
300-
return fieldType.parse(nullValue.getValue());
303+
final String fieldName = fieldType.name();
304+
return fieldType.parseNullValueWithDeprecation(nullValue.getValue(), fieldName, indexName);
301305
} catch (Exception e) {
302306
DEPRECATION_LOGGER.critical(DeprecationCategory.MAPPINGS, "date_mapper_null_field",
303307
"Error parsing [" + nullValue.getValue() + "] as date in [null_value] on field [" + name() + "]);"
@@ -311,7 +315,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
311315
DateFieldType ft = new DateFieldType(context.buildFullName(name()), index.getValue(), store.getValue(), docValues.getValue(),
312316
buildFormatter(), resolution, nullValue.getValue(), scriptValues(), meta.getValue());
313317
ft.setBoost(boost.getValue());
314-
Long nullTimestamp = parseNullValue(ft);
318+
Long nullTimestamp = parseNullValue(ft, indexName);
315319
return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context),
316320
copyTo.build(), nullTimestamp, resolution, this);
317321
}
@@ -320,13 +324,15 @@ public DateFieldMapper build(MapperBuilderContext context) {
320324
public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
321325
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
322326
return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), c.scriptCompiler(),
323-
ignoreMalformedByDefault, c.indexVersionCreated());
327+
ignoreMalformedByDefault, c.indexVersionCreated(),
328+
c.getIndexSettings() != null ? c.getIndexSettings().getIndex().getName() : null);
324329
});
325330

326331
public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
327332
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
328333
return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), c.scriptCompiler(),
329-
ignoreMalformedByDefault, c.indexVersionCreated());
334+
ignoreMalformedByDefault, c.indexVersionCreated(),
335+
c.getIndexSettings() != null ? c.getIndexSettings().getIndex().getName() : null);
330336
});
331337

332338
public static final class DateFieldType extends MappedFieldType {
@@ -382,7 +388,31 @@ protected DateMathParser dateMathParser() {
382388

383389
// Visible for testing.
384390
public long parse(String value) {
385-
return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant());
391+
final Instant instant = getInstant(value);
392+
return resolution.convert(instant);
393+
}
394+
395+
public long parseWithDeprecation(String value, String fieldName, String indexName) {
396+
final Instant instant = getInstant(value);
397+
if (resolution == Resolution.MILLISECONDS && instant.getNano() % 1000000 != 0) {
398+
DEPRECATION_LOGGER.warn(DeprecationCategory.MAPPINGS, "date_field_with_nanos",
399+
"You are attempting to store a nanosecond resolution on a field [{}] of type date on index [{}]. " +
400+
"The nanosecond part was lost. Use date_nanos field type.", fieldName, indexName);
401+
}
402+
return resolution.convert(instant);
403+
}
404+
405+
public long parseNullValueWithDeprecation(String value, String fieldName, String indexName) {
406+
final Instant instant = getInstant(value);
407+
if (resolution == Resolution.MILLISECONDS && instant.getNano() % 1000000 != 0) {
408+
DEPRECATION_LOGGER.warn(DeprecationCategory.MAPPINGS, "date_field_with_nanos",
409+
"You are attempting to set null_value with a nanosecond resolution on a field [{}] of type date on index [{}]. " +
410+
"The nanosecond part was lost. Use date_nanos field type.", fieldName, indexName);
411+
}
412+
return resolution.convert(instant);
413+
}
414+
private Instant getInstant(String value) {
415+
return DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant();
386416
}
387417

388418
/**
@@ -666,11 +696,13 @@ private DateFieldMapper(
666696
this.script = builder.script.get();
667697
this.scriptCompiler = builder.scriptCompiler;
668698
this.scriptValues = builder.scriptValues();
699+
this.indexName = builder.indexName;
669700
}
670701

671702
@Override
672703
public FieldMapper.Builder getMergeBuilder() {
673-
return new Builder(simpleName(), resolution, null, scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).init(this);
704+
return new Builder(simpleName(), resolution, null, scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexName)
705+
.init(this);
674706
}
675707

676708
@Override
@@ -695,7 +727,9 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
695727
timestamp = nullValue;
696728
} else {
697729
try {
698-
timestamp = fieldType().parse(dateAsString);
730+
final String fieldName = fieldType().name();
731+
final String indexName = context.indexSettings().getIndex().getName();
732+
timestamp = fieldType().parseWithDeprecation(dateAsString, fieldName, indexName);
699733
} catch (IllegalArgumentException | ElasticsearchParseException | DateTimeException | ArithmeticException e) {
700734
if (ignoreMalformed) {
701735
context.addIgnoredField(mappedFieldType.name());

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,8 @@ public void newDynamicDateField(DocumentParserContext context, String name, Date
300300
Settings settings = context.indexSettings().getSettings();
301301
boolean ignoreMalformed = FieldMapper.IGNORE_MALFORMED_SETTING.get(settings);
302302
createDynamicField(new DateFieldMapper.Builder(name, DateFieldMapper.Resolution.MILLISECONDS,
303-
dateTimeFormatter, ScriptCompiler.NONE, ignoreMalformed, context.indexSettings().getIndexVersionCreated()), context);
303+
dateTimeFormatter, ScriptCompiler.NONE, ignoreMalformed, context.indexSettings().getIndexVersionCreated(),
304+
context.indexSettings().getIndex().getName()), context);
304305
}
305306

306307
void newDynamicBinaryField(DocumentParserContext context, String name) throws IOException {

‎server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,7 @@ public Mapping parse(@Nullable String type, CompressedXContent source, String de
106106
}
107107

108108
MappingParserContext parserContext = parserContextSupplier.get();
109-
RootObjectMapper rootObjectMapper
110-
= rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT);
109+
RootObjectMapper rootObjectMapper = rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT);
111110

112111
Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers = metadataMappersFunction.apply(type);
113112
Map<String, Object> meta = null;

‎server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -565,15 +565,15 @@ public void testRolloverClusterStateForDataStream() throws Exception {
565565
null,
566566
ScriptCompiler.NONE,
567567
false,
568-
Version.CURRENT).build(MapperBuilderContext.ROOT);
568+
Version.CURRENT, "indexName").build(MapperBuilderContext.ROOT);
569569
ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool);
570570
Environment env = mock(Environment.class);
571571
when(env.sharedDataFile()).thenReturn(null);
572572
AllocationService allocationService = mock(AllocationService.class);
573573
when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]);
574574
RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc");
575575
root.add(new DateFieldMapper.Builder(dataStream.getTimeStampField().getName(), DateFieldMapper.Resolution.MILLISECONDS,
576-
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, true, Version.CURRENT));
576+
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, true, Version.CURRENT, "indexName"));
577577
MetadataFieldMapper dtfm = getDataStreamTimestampFieldMapper();
578578
Mapping mapping = new Mapping(
579579
root.build(MapperBuilderContext.ROOT),

‎server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,17 @@ public void testIgnoreMalformed() throws IOException {
148148
testIgnoreMalformedForValue("-522000000", "long overflow", "date_optional_time");
149149
}
150150

151+
public void testResolutionLossDeprecation() throws Exception {
152+
DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b
153+
.field("type", "date")));
154+
155+
ParsedDocument doc = mapper.parse(source(b -> b.field("field", "2018-10-03T14:42:44.123456+0000")));
156+
157+
assertWarnings("You are attempting to store a nanosecond resolution " +
158+
"on a field [field] of type date on index [index]. " +
159+
"The nanosecond part was lost. Use date_nanos field type.");
160+
}
161+
151162
private void testIgnoreMalformedForValue(String value, String expectedCause, String dateFormat) throws IOException {
152163

153164
DocumentMapper mapper = createDocumentMapper(fieldMapping((builder)-> dateFieldMapping(builder, dateFormat)));
@@ -403,11 +414,11 @@ public void testFetchMillisFromIso8601() throws IOException {
403414
}
404415

405416
public void testFetchMillisFromIso8601Nanos() throws IOException {
406-
assertFetch(dateMapperService(), "field", randomIs8601Nanos(MAX_ISO_DATE), null);
417+
assertFetch(dateNanosMapperService(), "field", randomIs8601Nanos(MAX_NANOS), null);
407418
}
408419

409420
public void testFetchMillisFromIso8601NanosFormatted() throws IOException {
410-
assertFetch(dateMapperService(), "field", randomIs8601Nanos(MAX_ISO_DATE), "strict_date_optional_time_nanos");
421+
assertFetch(dateNanosMapperService(), "field", randomIs8601Nanos(MAX_NANOS), "strict_date_optional_time_nanos");
411422
}
412423

413424
/**
@@ -418,7 +429,8 @@ public void testFetchMillisFromIso8601NanosFormatted() throws IOException {
418429
* way.
419430
*/
420431
public void testFetchMillisFromRoundedNanos() throws IOException {
421-
assertFetch(dateMapperService(), "field", randomDecimalNanos(MAX_ISO_DATE), null);
432+
assertFetch(dateMapperService(), "field", randomDecimalMillis(MAX_ISO_DATE), null);
433+
assertFetch(dateNanosMapperService(), "field", randomDecimalNanos(MAX_NANOS), null);
422434
}
423435

424436
/**
@@ -531,7 +543,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
531543
switch (((DateFieldType) ft).resolution()) {
532544
case MILLISECONDS:
533545
if (randomBoolean()) {
534-
return randomIs8601Nanos(MAX_ISO_DATE);
546+
return randomDecimalMillis(MAX_ISO_DATE);
535547
}
536548
return randomLongBetween(0, Long.MAX_VALUE);
537549
case NANOSECONDS:
@@ -564,6 +576,10 @@ private String randomDecimalNanos(long maxMillis) {
564576
return Long.toString(randomLongBetween(0, maxMillis)) + "." + between(0, 999999);
565577
}
566578

579+
private String randomDecimalMillis(long maxMillis) {
580+
return Long.toString(randomLongBetween(0, maxMillis));
581+
}
582+
567583
public void testScriptAndPrecludedParameters() {
568584
{
569585
Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {

‎x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.client.ResponseException;
1515
import org.elasticsearch.common.Strings;
1616
import org.elasticsearch.common.io.Streams;
17+
import org.elasticsearch.common.time.DateFormatter;
1718
import org.elasticsearch.xcontent.XContentBuilder;
1819
import org.elasticsearch.xcontent.json.JsonXContent;
1920
import org.elasticsearch.test.rest.ESRestTestCase;
@@ -101,7 +102,9 @@ private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws I
101102

102103
private Response indexDocument() throws IOException {
103104
final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc");
104-
indexDocumentRequest.setJsonEntity("{ \"@timestamp\": \"" + Instant.now() + "\", \"name\": \"Kimchi\" }");
105+
final Instant now = Instant.now();
106+
final String time = DateFormatter.forPattern("strict_date_optional_time").format(now);
107+
indexDocumentRequest.setJsonEntity("{ \"@timestamp\": \"" + time + "\", \"name\": \"Kimchi\" }");
105108
return client().performRequest(indexDocumentRequest);
106109
}
107110
}

‎x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
import org.elasticsearch.cluster.metadata.IndexMetadata;
1313
import org.elasticsearch.common.Strings;
1414
import org.elasticsearch.common.settings.Settings;
15+
import org.elasticsearch.common.time.DateFormatter;
16+
import org.elasticsearch.common.time.DateFormatters;
1517
import org.elasticsearch.test.rest.ESRestTestCase;
1618

1719
import java.time.ZoneOffset;
@@ -280,7 +282,7 @@ public void testHRDSplit() throws Exception {
280282

281283
for (int i = 1; i <= 100; i++) {
282284
ZonedDateTime time = baseTime.plusHours(i);
283-
String formattedTime = time.format(DateTimeFormatter.ISO_DATE_TIME);
285+
String formattedTime = DateFormatter.forPattern("strict_date_optional_time").format(time);
284286
if (i % 50 == 0) {
285287
// Anomaly has 100 docs, but we don't care about the value
286288
for (int j = 0; j < 100; j++) {

0 commit comments

Comments
 (0)