Configurable output format for date processor (#61324) (#62175)

Author: danhermann

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java

@@ -43,17 +43,24 @@ public final class DateProcessor extends AbstractProcessor {
 
     public static final String TYPE = "date";
     static final String DEFAULT_TARGET_FIELD = "@timestamp";
-    private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
+    static final String DEFAULT_OUTPUT_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX";
 
+    private final DateFormatter formatter;
     private final TemplateScript.Factory timezone;
     private final TemplateScript.Factory locale;
     private final String field;
     private final String targetField;
     private final List<String> formats;
     private final List<Function<Map<String, Object>, Function<String, ZonedDateTime>>> dateParsers;
+    private final String outputFormat;
 
     DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
                   String field, List<String> formats, String targetField) {
+        this(tag, description, timezone, locale, field, formats, targetField, DEFAULT_OUTPUT_FORMAT);
+    }
+
+    DateProcessor(String tag, String description, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
+                  String field, List<String> formats, String targetField, String outputFormat) {
         super(tag, description);
         this.timezone = timezone;
         this.locale = locale;
@@ -65,6 +72,8 @@ public final class DateProcessor extends AbstractProcessor {
             DateFormat dateFormat = DateFormat.fromString(format);
             dateParsers.add((params) -> dateFormat.getFunction(format, newDateTimeZone(params), newLocale(params)));
         }
+        this.outputFormat = outputFormat;
+        formatter = DateFormatter.forPattern(this.outputFormat);
     }
 
     private ZoneId newDateTimeZone(Map<String, Object> params) {
@@ -99,7 +108,7 @@ public IngestDocument execute(IngestDocument ingestDocument) {
             throw new IllegalArgumentException("unable to parse date [" + value + "]", lastException);
         }
 
-        ingestDocument.setFieldValue(targetField, FORMATTER.format(dateTime));
+        ingestDocument.setFieldValue(targetField, formatter.format(dateTime));
         return ingestDocument;
     }
 
@@ -128,6 +137,10 @@ List<String> getFormats() {
         return formats;
     }
 
+    String getOutputFormat() {
+        return outputFormat;
+    }
+
     public static final class Factory implements Processor.Factory {
 
         private final ScriptService scriptService;
@@ -153,8 +166,16 @@ public DateProcessor create(Map<String, Processor.Factory> registry, String proc
                     "locale", localeString, scriptService);
             }
             List<String> formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats");
+            String outputFormat =
+                ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "output_format", DEFAULT_OUTPUT_FORMAT);
+            try {
+                DateFormatter.forPattern(outputFormat);
+            } catch (Exception e) {
+                throw new IllegalArgumentException("invalid output format [" + outputFormat + "]", e);
+            }
+
             return new DateProcessor(processorTag, description, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats,
-                targetField);
+                targetField, outputFormat);
         }
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java

@@ -146,4 +146,42 @@ public void testParseTargetField() throws Exception {
         DateProcessor processor = factory.create(null, null, null, config);
         assertThat(processor.getTargetField(), equalTo(targetField));
     }
+
+    public void testParseOutputFormat() throws Exception {
+        final String outputFormat = "dd:MM:yyyy";
+        Map<String, Object> config = new HashMap<>();
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
+        String targetField = randomAlphaOfLengthBetween(1, 10);
+        config.put("field", sourceField);
+        config.put("target_field", targetField);
+        config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+        config.put("output_format", outputFormat);
+        DateProcessor processor = factory.create(null, null, null, config);
+        assertThat(processor.getOutputFormat(), equalTo(outputFormat));
+    }
+
+    public void testDefaultOutputFormat() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
+        String targetField = randomAlphaOfLengthBetween(1, 10);
+        config.put("field", sourceField);
+        config.put("target_field", targetField);
+        config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+        DateProcessor processor = factory.create(null, null, null, config);
+        assertThat(processor.getOutputFormat(), equalTo(DateProcessor.DEFAULT_OUTPUT_FORMAT));
+    }
+
+    public void testInvalidOutputFormatRejected() throws Exception {
+        final String outputFormat = "invalid_date_format";
+        Map<String, Object> config = new HashMap<>();
+        String sourceField = randomAlphaOfLengthBetween(1, 10);
+        String targetField = randomAlphaOfLengthBetween(1, 10);
+        config.put("field", sourceField);
+        config.put("target_field", targetField);
+        config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
+        config.put("output_format", outputFormat);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config));
+        assertThat(e.getMessage(), containsString("invalid output format [" + outputFormat + "]"));
+    }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.test.ESTestCase;
 
+import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
@@ -224,4 +225,17 @@ null, templatize(ZoneOffset.UTC), new TestTemplateService.MockTemplateScript.Fac
         assertThat(e.getMessage(), equalTo("unable to parse date [2010]"));
         assertThat(e.getCause().getMessage(), equalTo("Unknown language: invalid"));
     }
+
+    public void testOutputFormat() {
+        long nanosAfterEpoch = randomLongBetween(1, 999999);
+        DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), null, null, null,
+            "date_as_string", Collections.singletonList("iso8601"), "date_as_date", "HH:mm:ss.SSSSSSSSS");
+        Map<String, Object> document = new HashMap<>();
+        document.put("date_as_string", Instant.EPOCH.plusNanos(nanosAfterEpoch).toString());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+        // output format is time only with nanosecond precision
+        String expectedDate = "00:00:00." + String.format(Locale.ROOT, "%09d", nanosAfterEpoch);
+        assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(expectedDate));
+    }
 }