diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index 39be9ad8fb02a..a2bafd47d89f1 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -29,6 +29,7 @@ import java.time.ZonedDateTime; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; +import java.time.temporal.WeekFields; import java.util.Arrays; import java.util.List; import java.util.Locale; @@ -45,8 +46,12 @@ enum DateFormat { Iso8601 { @Override Function getFunction(String format, ZoneId timezone, Locale locale) { - return (date) -> DateFormatters.from(DateFormatter.forPattern("iso8601").parse(date), timezone) - .withZoneSameInstant(timezone); + return (date) -> { + TemporalAccessor accessor = DateFormatter.forPattern("iso8601").parse(date); + //even though locale could be set to en-us, Locale.ROOT (following iso8601 calendar data rules) should be used + return DateFormatters.from(accessor, Locale.ROOT, timezone) + .withZoneSameInstant(timezone); + }; } }, @@ -102,7 +107,9 @@ Function getFunction(String format, ZoneId zoneId, Locale TemporalAccessor accessor = formatter.parse(text); // if there is no year nor year-of-era, we fall back to the current one and // fill the rest of the date up with the parsed date - if (accessor.isSupported(ChronoField.YEAR) == false && accessor.isSupported(ChronoField.YEAR_OF_ERA) == false ) { + if (accessor.isSupported(ChronoField.YEAR) == false + && accessor.isSupported(ChronoField.YEAR_OF_ERA) == false + && accessor.isSupported(WeekFields.of(locale).weekOfWeekBasedYear()) == false) { int year = LocalDate.now(ZoneOffset.UTC).getYear(); ZonedDateTime newTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year); for (ChronoField field : FIELDS) { @@ -115,9 +122,9 @@ Function getFunction(String format, ZoneId zoneId, Locale } if (isUtc) { - return DateFormatters.from(accessor).withZoneSameInstant(ZoneOffset.UTC); + return DateFormatters.from(accessor, locale).withZoneSameInstant(ZoneOffset.UTC); } else { - return DateFormatters.from(accessor); + return DateFormatters.from(accessor, locale); } }; } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index 2325dfa37ab48..518ac20cec724 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -69,6 +69,23 @@ public void testParseJavaDefaultYear() { assertThat(dateTime.getYear(), is(year)); } + public void testParseWeekBased() { + String format = randomFrom("YYYY-ww"); + ZoneId timezone = DateUtils.of("Europe/Amsterdam"); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT); + ZonedDateTime dateTime = javaFunction.apply("2020-33"); + assertThat(dateTime, equalTo(ZonedDateTime.of(2020,8,10,0,0,0,0,timezone))); + } + + public void testParseWeekBasedWithLocale() { + String format = randomFrom("YYYY-ww"); + ZoneId timezone = DateUtils.of("Europe/Amsterdam"); + Function javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.US); + ZonedDateTime dateTime = javaFunction.apply("2020-33"); + //33rd week of 2020 starts on 9th August 2020 as per US locale + assertThat(dateTime, equalTo(ZonedDateTime.of(2020,8,9,0,0,0,0,timezone))); + } + public void testParseUnixMs() { assertThat(DateFormat.UnixMs.getFunction(null, ZoneOffset.UTC, null).apply("1000500").toInstant().toEpochMilli(), equalTo(1000500L)); diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml index f700e91e50b08..cee302f0e20ed 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -183,3 +183,132 @@ teardown: - match: { _source.date_source_7: "2018-02-05T13:44:56.657+0100" } - match: { _source.date_target_7: "2018-02-05T12:44:56.657Z" } + +--- +"Test week based date parsing": + - do: + indices.create: + index: test + body: + mappings: + properties: + date_source_field: + type: date + format: YYYY-ww + + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "date" : { + "field" : "date_source_field", + "target_field" : "date_target_field", + "formats" : ["YYYY-ww"] + } + } + ] + } + - match: { acknowledged: true } + + - do: + ingest.simulate: + id: "my_pipeline" + body: > + { + "docs": [ + { + "_source": { + "date_source_field": "2020-33" + } + } + ] + } + - length: { docs: 1 } + - match: { docs.0.doc._source.date_source_field: "2020-33" } + - match: { docs.0.doc._source.date_target_field: "2020-08-10T00:00:00.000Z" } + - length: { docs.0.doc._ingest: 1 } + - is_true: docs.0.doc._ingest.timestamp + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: {date_source_field: "2020-33"} + + - do: + get: + index: test + id: 1 + - match: { _source.date_source_field: "2020-33" } + - match: { _source.date_target_field: "2020-08-10T00:00:00.000Z" } + +--- +"Test week based date parsing with locale": + #locale is used when parsing as well on a pipeline. As per US locale, start of the 33rd week 2020 is on 09August2020 (sunday) + - do: + indices.create: + index: test + body: + mappings: + properties: + date_source_field: + type: date + format: YYYY-ww + locale: en-US + + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "date" : { + "field" : "date_source_field", + "target_field" : "date_target_field", + "formats" : ["YYYY-ww"], + "locale" : "en-US" + } + } + ] + } + - match: { acknowledged: true } + + - do: + ingest.simulate: + id: "my_pipeline" + body: > + { + "docs": [ + { + "_source": { + "date_source_field": "2020-33" + } + } + ] + } + - length: { docs: 1 } + - match: { docs.0.doc._source.date_source_field: "2020-33" } + - match: { docs.0.doc._source.date_target_field: "2020-08-09T00:00:00.000Z" } + - length: { docs.0.doc._ingest: 1 } + - is_true: docs.0.doc._ingest.timestamp + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: {date_source_field: "2020-33"} + + - do: + get: + index: test + id: 1 + - match: { _source.date_source_field: "2020-33" } + - match: { _source.date_target_field: "2020-08-09T00:00:00.000Z" } diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index 6f4a7a6cd7237..75bf85b07bdc0 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -36,6 +36,7 @@ import java.time.temporal.ChronoField; import java.time.temporal.IsoFields; import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalAdjusters; import java.time.temporal.TemporalQueries; import java.time.temporal.TemporalQuery; import java.time.temporal.WeekFields; @@ -51,7 +52,7 @@ import static java.time.temporal.ChronoField.SECOND_OF_MINUTE; public class DateFormatters { - public static final WeekFields WEEK_FIELDS = WeekFields.of(Locale.ROOT); + public static final WeekFields WEEK_FIELDS_ROOT = WeekFields.of(Locale.ROOT); private static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON = new DateTimeFormatterBuilder() .appendOffset("+HHmm", "Z") @@ -945,14 +946,14 @@ public class DateFormatters { * Returns a formatter for a four digit weekyear */ private static final DateFormatter STRICT_WEEKYEAR = new JavaDateFormatter("strict_weekyear", new DateTimeFormatterBuilder() - .appendValue(WEEK_FIELDS.weekBasedYear(), 4, 10, SignStyle.EXCEEDS_PAD) + .appendValue(WEEK_FIELDS_ROOT.weekBasedYear(), 4, 10, SignStyle.EXCEEDS_PAD) .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT)); private static final DateTimeFormatter STRICT_WEEKYEAR_WEEK_FORMATTER = new DateTimeFormatterBuilder() - .appendValue(WEEK_FIELDS.weekBasedYear(), 4, 10, SignStyle.EXCEEDS_PAD) + .appendValue(WEEK_FIELDS_ROOT.weekBasedYear(), 4, 10, SignStyle.EXCEEDS_PAD) .appendLiteral("-W") - .appendValue(WEEK_FIELDS.weekOfWeekBasedYear(), 2, 2, SignStyle.NOT_NEGATIVE) + .appendValue(WEEK_FIELDS_ROOT.weekOfWeekBasedYear(), 2, 2, SignStyle.NOT_NEGATIVE) .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT); @@ -971,7 +972,7 @@ public class DateFormatters { new DateTimeFormatterBuilder() .append(STRICT_WEEKYEAR_WEEK_FORMATTER) .appendLiteral("-") - .appendValue(WEEK_FIELDS.dayOfWeek()) + .appendValue(WEEK_FIELDS_ROOT.dayOfWeek()) .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT)); @@ -1161,7 +1162,7 @@ public class DateFormatters { * Returns a formatter for a four digit weekyear. (YYYY) */ private static final DateFormatter WEEK_YEAR = new JavaDateFormatter("week_year", - new DateTimeFormatterBuilder().appendValue(WEEK_FIELDS.weekBasedYear()).toFormatter(Locale.ROOT) + new DateTimeFormatterBuilder().appendValue(WEEK_FIELDS_ROOT.weekBasedYear()).toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT)); /* @@ -1590,9 +1591,9 @@ public class DateFormatters { */ private static final DateFormatter WEEKYEAR_WEEK = new JavaDateFormatter("weekyear_week", STRICT_WEEKYEAR_WEEK_FORMATTER, new DateTimeFormatterBuilder() - .appendValue(WEEK_FIELDS.weekBasedYear()) + .appendValue(WEEK_FIELDS_ROOT.weekBasedYear()) .appendLiteral("-W") - .appendValue(WEEK_FIELDS.weekOfWeekBasedYear()) + .appendValue(WEEK_FIELDS_ROOT.weekOfWeekBasedYear()) .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT) ); @@ -1605,15 +1606,15 @@ public class DateFormatters { new DateTimeFormatterBuilder() .append(STRICT_WEEKYEAR_WEEK_FORMATTER) .appendLiteral("-") - .appendValue(WEEK_FIELDS.dayOfWeek()) + .appendValue(WEEK_FIELDS_ROOT.dayOfWeek()) .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT), new DateTimeFormatterBuilder() - .appendValue(WEEK_FIELDS.weekBasedYear()) + .appendValue(WEEK_FIELDS_ROOT.weekBasedYear()) .appendLiteral("-W") - .appendValue(WEEK_FIELDS.weekOfWeekBasedYear()) + .appendValue(WEEK_FIELDS_ROOT.weekOfWeekBasedYear()) .appendLiteral("-") - .appendValue(WEEK_FIELDS.dayOfWeek()) + .appendValue(WEEK_FIELDS_ROOT.dayOfWeek()) .toFormatter(Locale.ROOT) .withResolverStyle(ResolverStyle.STRICT) ); @@ -1839,10 +1840,14 @@ public class DateFormatters { * @return The converted zoned date time */ public static ZonedDateTime from(TemporalAccessor accessor) { - return from(accessor, ZoneOffset.UTC); + return from(accessor, Locale.ROOT, ZoneOffset.UTC); } - public static ZonedDateTime from(TemporalAccessor accessor, ZoneId defaultZone) { + public static ZonedDateTime from(TemporalAccessor accessor, Locale locale) { + return from(accessor, locale, ZoneOffset.UTC); + } + + public static ZonedDateTime from(TemporalAccessor accessor, Locale locale, ZoneId defaultZone) { if (accessor instanceof ZonedDateTime) { return (ZonedDateTime) accessor; } @@ -1865,7 +1870,7 @@ public static ZonedDateTime from(TemporalAccessor accessor, ZoneId defaultZone) } else if (isLocalDateSet) { return localDate.atStartOfDay(zoneId); } else if (isLocalTimeSet) { - return of(getLocalDate(accessor), localTime, zoneId); + return of(getLocalDate(accessor, locale), localTime, zoneId); } else if (accessor.isSupported(ChronoField.YEAR) || accessor.isSupported(ChronoField.YEAR_OF_ERA) ) { if (accessor.isSupported(MONTH_OF_YEAR)) { return getFirstOfMonth(accessor).atStartOfDay(zoneId); @@ -1875,9 +1880,9 @@ public static ZonedDateTime from(TemporalAccessor accessor, ZoneId defaultZone) } } else if (accessor.isSupported(MONTH_OF_YEAR)) { // missing year, falling back to the epoch and then filling - return getLocalDate(accessor).atStartOfDay(zoneId); - } else if (accessor.isSupported(WEEK_FIELDS.weekBasedYear())) { - return localDateFromWeekBasedDate(accessor).atStartOfDay(zoneId); + return getLocalDate(accessor, locale).atStartOfDay(zoneId); + } else if (accessor.isSupported(WeekFields.of(locale).weekBasedYear())) { + return localDateFromWeekBasedDate(accessor, locale).atStartOfDay(zoneId); } // we should not reach this piece of code, everything being parsed we should be able to @@ -1885,16 +1890,18 @@ public static ZonedDateTime from(TemporalAccessor accessor, ZoneId defaultZone) throw new IllegalArgumentException("temporal accessor [" + accessor + "] cannot be converted to zoned date time"); } - private static LocalDate localDateFromWeekBasedDate(TemporalAccessor accessor) { - if (accessor.isSupported(WEEK_FIELDS.weekOfWeekBasedYear())) { + private static LocalDate localDateFromWeekBasedDate(TemporalAccessor accessor, Locale locale) { + WeekFields weekFields = WeekFields.of(locale); + if (accessor.isSupported(weekFields.weekOfWeekBasedYear())) { return LocalDate.ofEpochDay(0) - .with(WEEK_FIELDS.weekBasedYear(), accessor.get(WEEK_FIELDS.weekBasedYear())) - .with(WEEK_FIELDS.weekOfWeekBasedYear(), accessor.get(WEEK_FIELDS.weekOfWeekBasedYear())) - .with(ChronoField.DAY_OF_WEEK, WEEK_FIELDS.getFirstDayOfWeek().getValue()); + .with(weekFields.weekBasedYear(), accessor.get(weekFields.weekBasedYear())) + .with(weekFields.weekOfWeekBasedYear(), accessor.get(weekFields.weekOfWeekBasedYear())) + .with(TemporalAdjusters.previousOrSame(weekFields.getFirstDayOfWeek())); } else { return LocalDate.ofEpochDay(0) - .with(WEEK_FIELDS.weekBasedYear(), accessor.get(WEEK_FIELDS.weekBasedYear())) - .with(ChronoField.DAY_OF_WEEK, WEEK_FIELDS.getFirstDayOfWeek().getValue()); + .with(weekFields.weekBasedYear(), accessor.get(weekFields.weekBasedYear())) + .with(TemporalAdjusters.previousOrSame(weekFields.getFirstDayOfWeek())); + } } @@ -1925,9 +1932,9 @@ public String toString() { } }; - private static LocalDate getLocalDate(TemporalAccessor accessor) { - if (accessor.isSupported(WEEK_FIELDS.weekBasedYear())) { - return localDateFromWeekBasedDate(accessor); + private static LocalDate getLocalDate(TemporalAccessor accessor, Locale locale) { + if (accessor.isSupported(WeekFields.of(locale).weekBasedYear())) { + return localDateFromWeekBasedDate(accessor, locale); } else if (accessor.isSupported(MONTH_OF_YEAR)) { int year = getYear(accessor); if (accessor.isSupported(DAY_OF_MONTH)) { diff --git a/server/src/main/java/org/elasticsearch/common/time/IsoCalendarDataProvider.java b/server/src/main/java/org/elasticsearch/common/time/IsoCalendarDataProvider.java index 36a93049adeb8..324a2c0a2edf2 100644 --- a/server/src/main/java/org/elasticsearch/common/time/IsoCalendarDataProvider.java +++ b/server/src/main/java/org/elasticsearch/common/time/IsoCalendarDataProvider.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 98f222e822477..55f68ed4cbf21 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -357,7 +357,7 @@ protected DateMathParser dateMathParser() { } public long parse(String value) { - return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value)).toInstant()); + return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant()); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/JodaCompatibleZonedDateTime.java b/server/src/main/java/org/elasticsearch/script/JodaCompatibleZonedDateTime.java index 87d116265a697..03df6827aca10 100644 --- a/server/src/main/java/org/elasticsearch/script/JodaCompatibleZonedDateTime.java +++ b/server/src/main/java/org/elasticsearch/script/JodaCompatibleZonedDateTime.java @@ -477,13 +477,13 @@ public int getSecondOfMinute() { @Deprecated public int getWeekOfWeekyear() { logDeprecatedMethod("getWeekOfWeekyear()", "get(DateFormatters.WEEK_FIELDS.weekOfWeekBasedYear())"); - return dt.get(DateFormatters.WEEK_FIELDS.weekOfWeekBasedYear()); + return dt.get(DateFormatters.WEEK_FIELDS_ROOT.weekOfWeekBasedYear()); } @Deprecated public int getWeekyear() { logDeprecatedMethod("getWeekyear()", "get(DateFormatters.WEEK_FIELDS.weekBasedYear())"); - return dt.get(DateFormatters.WEEK_FIELDS.weekBasedYear()); + return dt.get(DateFormatters.WEEK_FIELDS_ROOT.weekBasedYear()); } @Deprecated