elastic · spinscale · Feb 9, 2019 · Feb 9, 2019
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
@@ -87,10 +87,16 @@ Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale
                 format = format.substring(1);
             }
 
+            boolean isUtc = ZoneOffset.UTC.equals(zoneId);
+
             int year = LocalDate.now(ZoneOffset.UTC).getYear();
-            DateFormatter formatter = DateFormatter.forPattern(format)
-                .withLocale(locale)
-                .withZone(zoneId);
+            DateFormatter dateFormatter = DateFormatter.forPattern(format)
+                .withLocale(locale);
+            // if UTC zone is set here, the the time zone specified in the format will be ignored, leading to wrong dates
+            if (isUtc == false) {
+                dateFormatter = dateFormatter.withZone(zoneId);
+            }
+            final DateFormatter formatter = dateFormatter;
             return text -> {
                 TemporalAccessor accessor = formatter.parse(text);
                 // if there is no year, we fall back to the current one and
@@ -106,7 +112,11 @@ Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale
                     accessor = newTime.withZoneSameLocal(zoneId);
                 }
 
-                return DateFormatters.from(accessor);
+                if (isUtc) {
+                    return DateFormatters.from(accessor).withZoneSameInstant(ZoneOffset.UTC);
+                } else {
+                    return DateFormatters.from(accessor);
+                }
             };
         }
     };

diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
@@ -43,7 +43,7 @@ public final class DateProcessor extends AbstractProcessor {
 
     public static final String TYPE = "date";
     static final String DEFAULT_TARGET_FIELD = "@timestamp";
-    public static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
+    private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
 
     private final TemplateScript.Factory timezone;
     private final TemplateScript.Factory locale;

diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.test.ESTestCase;
 
@@ -43,6 +44,14 @@ public void testParseJava() {
                 equalTo("11 24 01:29:01"));
     }
 
+    public void testParseJavaWithTimeZone() {
+        Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZZ",
+            ZoneOffset.UTC, Locale.ROOT);
+        ZonedDateTime datetime = javaFunction.apply("2018-02-05T13:44:56.657+0100");
+        String expectedDateTime = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").withZone(ZoneOffset.UTC).format(datetime);
+        assertThat(expectedDateTime, is("2018-02-05T12:44:56.657Z"));
+    }
+
     public void testParseJavaDefaultYear() {
         String format = randomFrom("8dd/MM", "dd/MM");
         ZoneId timezone = DateUtils.of("Europe/Amsterdam");
@@ -70,6 +79,10 @@ public void testParseUnixWithMsPrecision() {
     public void testParseISO8601() {
         assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(),
                 equalTo(978336000000L));
+        assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(),
+                equalTo("2001-01-01T08:00Z"));
+        assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(),
+                equalTo("2001-01-01T08:00Z"));
     }
 
     public void testParseISO8601Failure() {

diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
@@ -29,6 +29,7 @@
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -97,6 +98,18 @@ public void testJavaPatternMultipleFormats() {
         }
     }
 
+    public void testJavaPatternNoTimezone() {
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
+            null, null,
+            "date_as_string", Arrays.asList("yyyy dd MM HH:mm:ss XXX"), "date_as_date");
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("date_as_string", "2010 12 06 00:00:00 -02:00");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        dateProcessor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T02:00:00.000Z"));
+    }
+
     public void testInvalidJavaPattern() {
         try {
             DateProcessor processor = new DateProcessor(randomAlphaOfLength(10),

diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml
@@ -39,3 +39,94 @@ teardown:
         id: 1
   - match: { _source.date_source_field: "12/06/2010" }
   - match: { _source.date_target_field: "2010-06-12T00:00:00.000+02:00" }
+
+---
+"Test date processor with no timezone configured":
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        # sample formats from beats, featuring mongodb, icinga, apache
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "date" : {
+                  "field" : "date_source_1",
+                  "target_field" : "date_target_1",
+                  "formats" : ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_2",
+                  "target_field" : "date_target_2",
+                  "formats" : ["yyyy-MM-dd HH:mm:ss Z" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_3",
+                  "target_field" : "date_target_3",
+                  "formats" : [ "dd/MMM/yyyy:H:m:s Z" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_4",
+                  "target_field" : "date_target_4",
+                  "formats" : [ "UNIX" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_5",
+                  "target_field" : "date_target_5",
+                  "formats" : [ "UNIX_MS" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_6",
+                  "target_field" : "date_target_6",
+                  "formats" : [ "TAI64N" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_7",
+                  "target_field" : "date_target_7",
+                  "formats" : [ "ISO8601" ]
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { date_source_1: "2018-02-05T13:44:56.657+0100", date_source_2: "2017-04-04 13:43:09 +0200", date_source_3: "10/Aug/2018:09:45:56 +0200", date_source_4: "1", date_source_5: "1", date_source_6: "4000000050d506482dbdf024", date_source_7: "2018-02-05T13:44:56.657+0100" }
+
+  - do:
+      get:
+        index: test
+        id: 1
+  - match: { _source.date_source_1: "2018-02-05T13:44:56.657+0100" }
+  - match: { _source.date_target_1: "2018-02-05T12:44:56.657Z" }
+  - match: { _source.date_source_2: "2017-04-04 13:43:09 +0200" }
+  - match: { _source.date_target_2: "2017-04-04T11:43:09.000Z" }
+  - match: { _source.date_source_3: "10/Aug/2018:09:45:56 +0200" }
+  - match: { _source.date_target_3: "2018-08-10T07:45:56.000Z" }
+  - match: { _source.date_source_4: "1" }
+  - match: { _source.date_target_4: "1970-01-01T00:00:01.000Z" }
+  - match: { _source.date_source_5: "1" }
+  - match: { _source.date_target_5: "1970-01-01T00:00:00.001Z" }
+  - match: { _source.date_source_6: "4000000050d506482dbdf024" }
+  - match: { _source.date_target_6: "2012-12-22T01:00:46.767Z" }
+  - match: { _source.date_source_7: "2018-02-05T13:44:56.657+0100" }
+  - match: { _source.date_target_7: "2018-02-05T12:44:56.657Z" }
+