[6.8] Week based parsing for ingest date processor (#58597)

docs/reference/release-notes/6.8.asciidoc

@@ -34,6 +34,11 @@ This issue is fixed in Elasticsearch 6.8.10 and 7.7.1.
 
 Also see <<breaking-changes-6.8,Breaking changes in 6.8>>.
 
+* Java based - formats with '8' prefix - week based parsing and calculations are using JDK default calendar data provider which is Sunday,1.
+Sunday is considered first day of a week and it requires only 1 day in a week to for the first week of the year.
+It can be worked around by using locale which is based on ISO8601 rule (Monday,4) - for instance en-GB
+This issue is fixed in Elasticsearch 7.7 https://github.com/elastic/elasticsearch/pull/48209
+
 [[enhancement-6.8.9]]
 [float]
 === Enhancements

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java

@@ -33,6 +33,7 @@
 import java.time.ZonedDateTime;
 import java.time.temporal.ChronoField;
 import java.time.temporal.TemporalAccessor;
+import java.time.temporal.WeekFields;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
@@ -101,7 +102,10 @@ Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Loc
                     TemporalAccessor accessor = formatter.parse(text);
                     // if there is no year, we fall back to the current one and
                     // fill the rest of the date up with the parsed date
-                    if (accessor.isSupported(ChronoField.YEAR) == false) {
+                    if (accessor.isSupported(ChronoField.YEAR) == false
+                        && accessor.isSupported(ChronoField.YEAR_OF_ERA) == false
+                        && accessor.isSupported(WeekFields.of(locale).weekOfWeekBasedYear()) == false) {
+
                         ZonedDateTime newTime = Instant.EPOCH.atZone(ZoneOffset.UTC).withYear(year);
                         for (ChronoField field : FIELDS) {
                             if (accessor.isSupported(field)) {
@@ -112,7 +116,7 @@ Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Loc
                         accessor = newTime.withZoneSameLocal(DateUtils.dateTimeZoneToZoneId(timezone));
                     }
 
-                    long millis = DateFormatters.from(accessor).toInstant().toEpochMilli();
+                    long millis = DateFormatters.from(accessor, locale).toInstant().toEpochMilli();
                     return new DateTime(millis, timezone);
                 };
             } else {

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java

@@ -77,6 +77,32 @@ public void testParseDefaultYearBackwardsCompatible() {
         assertThat(nextYear, is(jodaDateTime.withZone(DateTimeZone.UTC).getYear()));
     }
 
+    public void testParseWeekBased() {
+        String format = "8YYYY-ww";
+        ZoneId zoneId = ZoneId.of("Europe/Amsterdam");
+        DateTimeZone timezone = DateUtils.zoneIdToDateTimeZone(zoneId);
+
+        Function<String, DateTime> javaFunction = DateFormat.Java.getFunction(format, timezone, Locale.ROOT);
+        DateTime dateTime = javaFunction.apply("2020-33");
+        assertThat(dateTime, equalTo(new DateTime(2020,8,9,0,0,0,0,timezone)));
+    }
+
+    public void testParseWeekBasedWithLocale() {
+        String format = "8YYYY-ww";
+        ZoneId zoneId = ZoneId.of("Europe/Amsterdam");
+        DateTimeZone timezone = DateUtils.zoneIdToDateTimeZone(zoneId);
+
+        Function<String, DateTime> javaFunctionUS = DateFormat.Java.getFunction(format, timezone, Locale.US);
+        DateTime dateTime = javaFunctionUS.apply("2020-33");
+        //33rd week of 2020 starts on 9th August 2020 as per US locale
+        assertThat(dateTime, equalTo(new DateTime(2020,8,9,0,0,0,0,timezone)));
+
+        Function<String, DateTime> javaFunctionUK = DateFormat.Java.getFunction(format, timezone, Locale.UK);
+        dateTime = javaFunctionUK.apply("2020-33");
+        //33rd week of 2020 starts on 10th August 2020 as per UK locale
+        assertThat(dateTime, equalTo(new DateTime(2020,8,10,0,0,0,0,timezone)));
+    }
+
     public void testParseUnixMs() {
         assertThat(DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null).apply("1000500").getMillis(), equalTo(1000500L));
     }

modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml

@@ -42,3 +42,208 @@ teardown:
   - match: { _source.date_source_field: "12/06/2010" }
   - match: { _source.date_target_field: "2010-06-12T00:00:00.000+02:00" }
 
+---
+"Test week based date parsing with default locale - Sunday,1 rule":
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            test:
+              properties:
+                date_source_field:
+                  type: date
+                  format: 8YYYY-ww
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "date" : {
+                  "field" : "date_source_field",
+                  "target_field" : "date_target_field",
+                  "formats" : ["8YYYY-ww"]
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.simulate:
+        id: "my_pipeline"
+        body: >
+          {
+            "docs": [
+              {
+                "_source": {
+                  "date_source_field": "2020-33"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._source.date_source_field: "2020-33" }
+  #we don't have a support for changing calendar data provider in 6.8. Hence it is using Sunday,1 (java default)
+  - match: { docs.0.doc._source.date_target_field: "2020-08-09T00:00:00.000Z" }
+  - length: { docs.0.doc._ingest: 1 }
+  - is_true: docs.0.doc._ingest.timestamp
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {date_source_field: "2020-33"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.date_source_field: "2020-33" }
+#we don't have a support for changing calendar data provider in 6.8. Hence it is using Sunday,1 (java default)
+  - match: { _source.date_target_field: "2020-08-09T00:00:00.000Z" }
+
+---
+"Test week based date parsing with ISO based locale - Monday,4 rule":
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            test:
+              properties:
+                date_source_field:
+                  type: date
+                  format: 8YYYY-ww
+                  locale: en-GB
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "date" : {
+                  "field" : "date_source_field",
+                  "target_field" : "date_target_field",
+                  "formats" : ["8YYYY-ww"],
+                  "locale" : "en-GB"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.simulate:
+        id: "my_pipeline"
+        body: >
+          {
+            "docs": [
+              {
+                "_source": {
+                  "date_source_field": "2020-33"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._source.date_source_field: "2020-33" }
+  - match: { docs.0.doc._source.date_target_field: "2020-08-10T00:00:00.000Z" }
+  - length: { docs.0.doc._ingest: 1 }
+  - is_true: docs.0.doc._ingest.timestamp
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {date_source_field: "2020-33"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.date_source_field: "2020-33" }
+  - match: { _source.date_target_field: "2020-08-10T00:00:00.000Z" }
+
+
+---
+"Test week based date parsing with locale":
+  #locale is used when parsing as well on a pipeline. As per US locale, start of the 33rd week 2020 is on 09August2020 (sunday)
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            test:
+              properties:
+                date_source_field:
+                  type: date
+                  format: 8YYYY-ww
+                  locale: en-US
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "date" : {
+                  "field" : "date_source_field",
+                  "target_field" : "date_target_field",
+                  "formats" : ["8YYYY-ww"],
+                  "locale" : "en-US"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.simulate:
+        id: "my_pipeline"
+        body: >
+          {
+            "docs": [
+              {
+                "_source": {
+                  "date_source_field": "2020-33"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._source.date_source_field: "2020-33" }
+  - match: { docs.0.doc._source.date_target_field: "2020-08-09T00:00:00.000Z" }
+  - length: { docs.0.doc._ingest: 1 }
+  - is_true: docs.0.doc._ingest.timestamp
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {date_source_field: "2020-33"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.date_source_field: "2020-33" }
+  - match: { _source.date_target_field: "2020-08-09T00:00:00.000Z" }