Add BWC compatible processing to ingest date processors

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java

@@ -19,12 +19,19 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.DateFormatters;
+import org.elasticsearch.common.time.DateUtils;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
 import org.joda.time.format.ISODateTimeFormat;
 
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.temporal.TemporalAccessor;
 import java.util.Locale;
 import java.util.function.Function;
 
@@ -63,11 +70,33 @@ private long parseMillis(String date) {
             return ((base * 1000) - 10000) + (rest/1000000);
         }
     },
-    Joda {
+    Java {
         @Override
         Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
-            DateTimeFormatter parser = DateTimeFormat.forPattern(format).withZone(timezone).withLocale(locale);
-            return text -> parser.withDefaultYear((new DateTime(DateTimeZone.UTC)).getYear()).parseDateTime(text);
+            // in case you are wondering why we do not call 'DateFormatter.forPattern(format)' for all cases here, but only for the
+            // non java time case:
+            // When the joda date formatter parses a date then a year is always set, so that no fallback can be used, like
+            // done in the JodaDateFormatter.withYear() code below
+            // This means that we leave the existing parsing logic in place, but will fall back to the new java date parsing logic, if an
+            // "8" is prepended to the date format string
+            int year = LocalDate.now(ZoneOffset.UTC).getYear();
+            if (format.startsWith("8")) {
+                DateFormatter formatter = DateFormatter.forPattern(format)
+                    .withLocale(locale)
+                    .withZone(DateUtils.dateTimeZoneToZoneId(timezone));
+                return text -> {
+                    ZonedDateTime defaultZonedDateTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year);
+                    TemporalAccessor accessor = formatter.parse(text);
+                    long millis = DateFormatters.toZonedDateTime(accessor, defaultZonedDateTime).toInstant().toEpochMilli();
+                    return new DateTime(millis, timezone);
+                };
+            } else {
+                DateFormatter formatter = Joda.forPattern(format)
+                    .withYear(year)
+                    .withZone(DateUtils.dateTimeZoneToZoneId(timezone))
+                    .withLocale(locale);
+                return text -> new DateTime(formatter.parseMillis(text), timezone);
+            }
         }
     };
 
@@ -84,7 +113,7 @@ static DateFormat fromString(String format) {
             case "TAI64N":
                 return Tai64n;
             default:
-                return Joda;
+                return Java;
         }
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java

@@ -34,7 +34,7 @@
 public class DateFormatTests extends ESTestCase {
 
     public void testParseJoda() {
-        Function<String, DateTime> jodaFunction = DateFormat.Joda.getFunction("MMM dd HH:mm:ss Z",
+        Function<String, DateTime> jodaFunction = DateFormat.Java.getFunction("MMM dd HH:mm:ss Z",
                 DateTimeZone.forOffsetHours(-8), Locale.ENGLISH);
         assertThat(Instant.ofEpochMilli(jodaFunction.apply("Nov 24 01:29:01 -0800").getMillis())
                         .atZone(ZoneId.of("GMT-8"))
@@ -78,13 +78,13 @@ public void testTAI64NParse() {
 
     public void testFromString() {
         assertThat(DateFormat.fromString("UNIX_MS"), equalTo(DateFormat.UnixMs));
-        assertThat(DateFormat.fromString("unix_ms"), equalTo(DateFormat.Joda));
+        assertThat(DateFormat.fromString("unix_ms"), equalTo(DateFormat.Java));
         assertThat(DateFormat.fromString("UNIX"), equalTo(DateFormat.Unix));
-        assertThat(DateFormat.fromString("unix"), equalTo(DateFormat.Joda));
+        assertThat(DateFormat.fromString("unix"), equalTo(DateFormat.Java));
         assertThat(DateFormat.fromString("ISO8601"), equalTo(DateFormat.Iso8601));
-        assertThat(DateFormat.fromString("iso8601"), equalTo(DateFormat.Joda));
+        assertThat(DateFormat.fromString("iso8601"), equalTo(DateFormat.Java));
         assertThat(DateFormat.fromString("TAI64N"), equalTo(DateFormat.Tai64n));
-        assertThat(DateFormat.fromString("tai64n"), equalTo(DateFormat.Joda));
-        assertThat(DateFormat.fromString("prefix-" + randomAlphaOfLengthBetween(1, 10)), equalTo(DateFormat.Joda));
+        assertThat(DateFormat.fromString("tai64n"), equalTo(DateFormat.Java));
+        assertThat(DateFormat.fromString("prefix-" + randomAlphaOfLengthBetween(1, 10)), equalTo(DateFormat.Java));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java

@@ -35,7 +35,7 @@
 public class DateIndexNameProcessorTests extends ESTestCase {
 
     public void testJodaPattern() throws Exception {
-        Function<String, DateTime> function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
+        Function<String, DateTime> function = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
         DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function),
             DateTimeZone.UTC, "events-", "y", "yyyyMMdd");
         IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java

@@ -109,7 +109,7 @@ public void testInvalidJodaPattern() {
             fail("date processor execution should have failed");
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("unable to parse date [2010]"));
-            assertThat(e.getCause().getMessage(), equalTo("Illegal pattern component: i"));
+            assertThat(e.getCause().getMessage(), equalTo("Invalid format: [invalid pattern]: Illegal pattern component: i"));
         }
     }
 
@@ -127,9 +127,10 @@ public void testJodaPatternLocale() {
     }
 
     public void testJodaPatternDefaultYear() {
+        String format = randomFrom("dd/MM", "8dd/MM");
         DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
             templatize(ZoneId.of("Europe/Amsterdam")), templatize(Locale.ENGLISH),
-            "date_as_string", Collections.singletonList("dd/MM"), "date_as_date");
+            "date_as_string", Collections.singletonList(format), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "12/06");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);

qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yml

@@ -1,5 +1,10 @@
 ---
 "Test logging":
+  - skip:
+      version: " - 6.9.99"
+      reason: pre-7.0.0 will send no warnings
+      features: "warnings"
+
   - do:
       ingest.put_pipeline:
         id: "_id"
@@ -41,6 +46,8 @@
   - match: { acknowledged: true }
 
   - do:
+      warnings:
+        - "Use of 'Y' (year-of-era) will change to 'y' in the next major version of Elasticsearch. Prefix your date format with '8' to use the new specifier."
       index:
         index: test
         type: test

server/src/main/java/org/elasticsearch/common/joda/Joda.java

@@ -107,8 +107,8 @@ public static JodaDateFormatter forPattern(String input) {
             // in this case, we have a separate parser and printer since the dataOptionalTimeParser can't print
             // this sucks we should use the root local by default and not be dependent on the node
             return new JodaDateFormatter(input,
-                    ISODateTimeFormat.dateOptionalTimeParser().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC),
-                    ISODateTimeFormat.dateTime().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC));
+                    ISODateTimeFormat.dateOptionalTimeParser().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970),
+                    ISODateTimeFormat.dateTime().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970));
         } else if ("dateTime".equals(input) || "date_time".equals(input)) {
             formatter = ISODateTimeFormat.dateTime();
         } else if ("dateTimeNoMillis".equals(input) || "date_time_no_millis".equals(input)) {
@@ -184,8 +184,9 @@ public static JodaDateFormatter forPattern(String input) {
             // in this case, we have a separate parser and printer since the dataOptionalTimeParser can't print
             // this sucks we should use the root local by default and not be dependent on the node
             return new JodaDateFormatter(input,
-                    StrictISODateTimeFormat.dateOptionalTimeParser().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC),
-                    StrictISODateTimeFormat.dateTime().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC));
+                    StrictISODateTimeFormat.dateOptionalTimeParser().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC)
+                        .withDefaultYear(1970),
+                    StrictISODateTimeFormat.dateTime().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970));
         } else if ("strictDateTime".equals(input) || "strict_date_time".equals(input)) {
             formatter = StrictISODateTimeFormat.dateTime();
         } else if ("strictDateTimeNoMillis".equals(input) || "strict_date_time_no_millis".equals(input)) {
@@ -262,7 +263,7 @@ public static JodaDateFormatter forPattern(String input) {
             }
         }
 
-        formatter = formatter.withLocale(Locale.ROOT).withZone(DateTimeZone.UTC);
+        formatter = formatter.withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970);
         return new JodaDateFormatter(input, formatter, formatter);
     }
 
@@ -311,7 +312,7 @@ public static DateFormatter getStrictStandardDateFormatter() {
         DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder().append(longFormatter.withZone(DateTimeZone.UTC).getPrinter(),
             new DateTimeParser[]{longFormatter.getParser(), shortFormatter.getParser(), new EpochTimeParser(true)});
 
-        DateTimeFormatter formatter = builder.toFormatter().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC);
+        DateTimeFormatter formatter = builder.toFormatter().withLocale(Locale.ROOT).withZone(DateTimeZone.UTC).withDefaultYear(1970);
         return new JodaDateFormatter("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||epoch_millis", formatter, formatter);
     }
 

server/src/main/java/org/elasticsearch/common/joda/JodaDateFormatter.java

@@ -39,10 +39,10 @@ public class JodaDateFormatter implements DateFormatter {
     final DateTimeFormatter parser;
     final DateTimeFormatter printer;
 
-    public JodaDateFormatter(String pattern, DateTimeFormatter parser, DateTimeFormatter printer) {
+    JodaDateFormatter(String pattern, DateTimeFormatter parser, DateTimeFormatter printer) {
         this.pattern = pattern;
-        this.printer = printer.withDefaultYear(1970);
-        this.parser = parser.withDefaultYear(1970);
+        this.printer = printer;
+        this.parser = parser;
     }
 
     @Override
@@ -62,13 +62,19 @@ public DateTime parseJoda(String input) {
     @Override
     public DateFormatter withZone(ZoneId zoneId) {
         DateTimeZone timeZone = DateUtils.zoneIdToDateTimeZone(zoneId);
+        if (parser.getZone().equals(timeZone)) {
+            return this;
+        }
         DateTimeFormatter parser = this.parser.withZone(timeZone);
         DateTimeFormatter printer = this.printer.withZone(timeZone);
         return new JodaDateFormatter(pattern, parser, printer);
     }
 
     @Override
     public DateFormatter withLocale(Locale locale) {
+        if (parser.getLocale().equals(locale)) {
+            return this;
+        }
         DateTimeFormatter parser = this.parser.withLocale(locale);
         DateTimeFormatter printer = this.printer.withLocale(locale);
         return new JodaDateFormatter(pattern, parser, printer);
@@ -89,6 +95,13 @@ public String formatMillis(long millis) {
         return printer.print(millis);
     }
 
+    public JodaDateFormatter withYear(int year) {
+        if (parser.getDefaultYear() == year) {
+            return this;
+        }
+        return new JodaDateFormatter(pattern, parser.withDefaultYear(year), printer.withDefaultYear(year));
+    }
+
     @Override
     public String pattern() {
         return pattern;