Skip to content

Commit d22fd4e

Browse files
talevytalevy
authored
Introduce templating support to timezone/locale in DateProcessor (#27089)
Sometimes systems like Beats would want to extract the date's timezone and/or locale from a value in a field of the document. This PR adds support for mustache templating to extract these values. Closes #24024.
1 parent e04e5ab commit d22fd4e

File tree

5 files changed

+128
-79
lines changed

5 files changed

+128
-79
lines changed

‎docs/reference/ingest/ingest-node.asciidoc‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -852,6 +852,30 @@ Here is an example that adds the parsed date to the `timestamp` field based on t
852852
--------------------------------------------------
853853
// NOTCONSOLE
854854

855+
The `timezone` and `locale` processor parameters are templated. This means that their values can be
856+
extracted from fields within documents. The example below shows how to extract the locale/timezone
857+
details from existing fields, `my_timezone` and `my_locale`, in the ingested document that contain
858+
the timezone and locale values.
859+
860+
[source,js]
861+
--------------------------------------------------
862+
{
863+
"description" : "...",
864+
"processors" : [
865+
{
866+
"date" : {
867+
"field" : "initial_date",
868+
"target_field" : "timestamp",
869+
"formats" : ["ISO8601"],
870+
"timezone" : "{{ my_timezone }}",
871+
"locale" : "{{ my_locale }}"
872+
}
873+
}
874+
]
875+
}
876+
--------------------------------------------------
877+
// NOTCONSOLE
878+
855879
[[date-index-name-processor]]
856880
=== Date Index Name Processor
857881

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,14 @@
2020
package org.elasticsearch.ingest.common;
2121

2222
import org.elasticsearch.ExceptionsHelper;
23+
import org.elasticsearch.common.Nullable;
2324
import org.elasticsearch.common.util.LocaleUtils;
2425
import org.elasticsearch.ingest.AbstractProcessor;
2526
import org.elasticsearch.ingest.ConfigurationUtils;
2627
import org.elasticsearch.ingest.IngestDocument;
2728
import org.elasticsearch.ingest.Processor;
29+
import org.elasticsearch.script.ScriptService;
30+
import org.elasticsearch.script.TemplateScript;
2831
import org.joda.time.DateTime;
2932
import org.joda.time.DateTimeZone;
3033
import org.joda.time.format.ISODateTimeFormat;
@@ -40,14 +43,15 @@ public final class DateProcessor extends AbstractProcessor {
4043
public static final String TYPE = "date";
4144
static final String DEFAULT_TARGET_FIELD = "@timestamp";
4245

43-
private final DateTimeZone timezone;
44-
private final Locale locale;
46+
private final TemplateScript.Factory timezone;
47+
private final TemplateScript.Factory locale;
4548
private final String field;
4649
private final String targetField;
4750
private final List<String> formats;
48-
private final List<Function<String, DateTime>> dateParsers;
51+
private final List<Function<Map<String, Object>, Function<String, DateTime>>> dateParsers;
4952

50-
DateProcessor(String tag, DateTimeZone timezone, Locale locale, String field, List<String> formats, String targetField) {
53+
DateProcessor(String tag, @Nullable TemplateScript.Factory timezone, @Nullable TemplateScript.Factory locale,
54+
String field, List<String> formats, String targetField) {
5155
super(tag);
5256
this.timezone = timezone;
5357
this.locale = locale;
@@ -57,10 +61,18 @@ public final class DateProcessor extends AbstractProcessor {
5761
this.dateParsers = new ArrayList<>(this.formats.size());
5862
for (String format : formats) {
5963
DateFormat dateFormat = DateFormat.fromString(format);
60-
dateParsers.add(dateFormat.getFunction(format, timezone, locale));
64+
dateParsers.add((params) -> dateFormat.getFunction(format, newDateTimeZone(params), newLocale(params)));
6165
}
6266
}
6367

68+
private DateTimeZone newDateTimeZone(Map<String, Object> params) {
69+
return timezone == null ? DateTimeZone.UTC : DateTimeZone.forID(timezone.newInstance(params).execute());
70+
}
71+
72+
private Locale newLocale(Map<String, Object> params) {
73+
return (locale == null) ? Locale.ROOT : LocaleUtils.parse(locale.newInstance(params).execute());
74+
}
75+
6476
@Override
6577
public void execute(IngestDocument ingestDocument) {
6678
Object obj = ingestDocument.getFieldValue(field, Object.class);
@@ -72,9 +84,9 @@ public void execute(IngestDocument ingestDocument) {
7284

7385
DateTime dateTime = null;
7486
Exception lastException = null;
75-
for (Function<String, DateTime> dateParser : dateParsers) {
87+
for (Function<Map<String, Object>, Function<String, DateTime>> dateParser : dateParsers) {
7688
try {
77-
dateTime = dateParser.apply(value);
89+
dateTime = dateParser.apply(ingestDocument.getSourceAndMetadata()).apply(value);
7890
} catch (Exception e) {
7991
//try the next parser and keep track of the exceptions
8092
lastException = ExceptionsHelper.useOrSuppress(lastException, e);
@@ -93,11 +105,11 @@ public String getType() {
93105
return TYPE;
94106
}
95107

96-
DateTimeZone getTimezone() {
108+
TemplateScript.Factory getTimezone() {
97109
return timezone;
98110
}
99111

100-
Locale getLocale() {
112+
TemplateScript.Factory getLocale() {
101113
return locale;
102114
}
103115

@@ -115,19 +127,30 @@ List<String> getFormats() {
115127

116128
public static final class Factory implements Processor.Factory {
117129

130+
private final ScriptService scriptService;
131+
132+
public Factory(ScriptService scriptService) {
133+
this.scriptService = scriptService;
134+
}
135+
118136
public DateProcessor create(Map<String, Processor.Factory> registry, String processorTag,
119137
Map<String, Object> config) throws Exception {
120138
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
121139
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD);
122140
String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "timezone");
123-
DateTimeZone timezone = timezoneString == null ? DateTimeZone.UTC : DateTimeZone.forID(timezoneString);
141+
TemplateScript.Factory compiledTimezoneTemplate = null;
142+
if (timezoneString != null) {
143+
compiledTimezoneTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
144+
"timezone", timezoneString, scriptService);
145+
}
124146
String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "locale");
125-
Locale locale = Locale.ROOT;
147+
TemplateScript.Factory compiledLocaleTemplate = null;
126148
if (localeString != null) {
127-
locale = LocaleUtils.parse(localeString);
149+
compiledLocaleTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag,
150+
"locale", localeString, scriptService);
128151
}
129152
List<String> formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats");
130-
return new DateProcessor(processorTag, timezone, locale, field, formats, targetField);
153+
return new DateProcessor(processorTag, compiledTimezoneTemplate, compiledLocaleTemplate, field, formats, targetField);
131154
}
132155
}
133156
}

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ public IngestCommonPlugin() throws IOException {
7070
@Override
7171
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
7272
Map<String, Processor.Factory> processors = new HashMap<>();
73-
processors.put(DateProcessor.TYPE, new DateProcessor.Factory());
73+
processors.put(DateProcessor.TYPE, new DateProcessor.Factory(parameters.scriptService));
7474
processors.put(SetProcessor.TYPE, new SetProcessor.Factory(parameters.scriptService));
7575
processors.put(AppendProcessor.TYPE, new AppendProcessor.Factory(parameters.scriptService));
7676
processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory());

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java‎

Lines changed: 13 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@
2020
package org.elasticsearch.ingest.common;
2121

2222
import org.elasticsearch.ElasticsearchParseException;
23+
import org.elasticsearch.ingest.TestTemplateService;
2324
import org.elasticsearch.test.ESTestCase;
2425
import org.joda.time.DateTimeZone;
26+
import org.junit.Before;
2527

2628
import java.util.Arrays;
2729
import java.util.Collections;
@@ -34,8 +36,14 @@
3436

3537
public class DateProcessorFactoryTests extends ESTestCase {
3638

39+
private DateProcessor.Factory factory;
40+
41+
@Before
42+
public void init() {
43+
factory = new DateProcessor.Factory(TestTemplateService.instance());
44+
}
45+
3746
public void testBuildDefaults() throws Exception {
38-
DateProcessor.Factory factory = new DateProcessor.Factory();
3947
Map<String, Object> config = new HashMap<>();
4048
String sourceField = randomAlphaOfLengthBetween(1, 10);
4149
config.put("field", sourceField);
@@ -46,12 +54,11 @@ public void testBuildDefaults() throws Exception {
4654
assertThat(processor.getField(), equalTo(sourceField));
4755
assertThat(processor.getTargetField(), equalTo(DateProcessor.DEFAULT_TARGET_FIELD));
4856
assertThat(processor.getFormats(), equalTo(Collections.singletonList("dd/MM/yyyyy")));
49-
assertThat(processor.getLocale(), equalTo(Locale.ROOT));
50-
assertThat(processor.getTimezone(), equalTo(DateTimeZone.UTC));
57+
assertNull(processor.getLocale());
58+
assertNull(processor.getTimezone());
5159
}
5260

5361
public void testMatchFieldIsMandatory() throws Exception {
54-
DateProcessor.Factory factory = new DateProcessor.Factory();
5562
Map<String, Object> config = new HashMap<>();
5663
String targetField = randomAlphaOfLengthBetween(1, 10);
5764
config.put("target_field", targetField);
@@ -66,7 +73,6 @@ public void testMatchFieldIsMandatory() throws Exception {
6673
}
6774

6875
public void testMatchFormatsIsMandatory() throws Exception {
69-
DateProcessor.Factory factory = new DateProcessor.Factory();
7076
Map<String, Object> config = new HashMap<>();
7177
String sourceField = randomAlphaOfLengthBetween(1, 10);
7278
String targetField = randomAlphaOfLengthBetween(1, 10);
@@ -82,7 +88,6 @@ public void testMatchFormatsIsMandatory() throws Exception {
8288
}
8389

8490
public void testParseLocale() throws Exception {
85-
DateProcessor.Factory factory = new DateProcessor.Factory();
8691
Map<String, Object> config = new HashMap<>();
8792
String sourceField = randomAlphaOfLengthBetween(1, 10);
8893
config.put("field", sourceField);
@@ -91,39 +96,10 @@ public void testParseLocale() throws Exception {
9196
config.put("locale", locale.toLanguageTag());
9297

9398
DateProcessor processor = factory.create(null, null, config);
94-
assertThat(processor.getLocale().toLanguageTag(), equalTo(locale.toLanguageTag()));
95-
}
96-
97-
public void testParseInvalidLocale() throws Exception {
98-
String[] locales = new String[] { "invalid_locale", "english", "xy", "xy-US" };
99-
for (String locale : locales) {
100-
DateProcessor.Factory factory = new DateProcessor.Factory();
101-
Map<String, Object> config = new HashMap<>();
102-
String sourceField = randomAlphaOfLengthBetween(1, 10);
103-
config.put("field", sourceField);
104-
config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
105-
config.put("locale", locale);
106-
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
107-
() -> factory.create(null, null, config));
108-
assertThat(e.getMessage(), equalTo("Unknown language: " + locale.split("[_-]")[0]));
109-
}
110-
111-
locales = new String[] { "en-XY", "en-Canada" };
112-
for (String locale : locales) {
113-
DateProcessor.Factory factory = new DateProcessor.Factory();
114-
Map<String, Object> config = new HashMap<>();
115-
String sourceField = randomAlphaOfLengthBetween(1, 10);
116-
config.put("field", sourceField);
117-
config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
118-
config.put("locale", locale);
119-
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
120-
() -> factory.create(null, null, config));
121-
assertThat(e.getMessage(), equalTo("Unknown country: " + locale.split("[_-]")[1]));
122-
}
99+
assertThat(processor.getLocale().newInstance(Collections.emptyMap()).execute(), equalTo(locale.toLanguageTag()));
123100
}
124101

125102
public void testParseTimezone() throws Exception {
126-
DateProcessor.Factory factory = new DateProcessor.Factory();
127103
Map<String, Object> config = new HashMap<>();
128104
String sourceField = randomAlphaOfLengthBetween(1, 10);
129105
config.put("field", sourceField);
@@ -132,26 +108,10 @@ public void testParseTimezone() throws Exception {
132108
DateTimeZone timezone = randomDateTimeZone();
133109
config.put("timezone", timezone.getID());
134110
DateProcessor processor = factory.create(null, null, config);
135-
assertThat(processor.getTimezone(), equalTo(timezone));
136-
}
137-
138-
public void testParseInvalidTimezone() throws Exception {
139-
DateProcessor.Factory factory = new DateProcessor.Factory();
140-
Map<String, Object> config = new HashMap<>();
141-
String sourceField = randomAlphaOfLengthBetween(1, 10);
142-
config.put("field", sourceField);
143-
config.put("match_formats", Collections.singletonList("dd/MM/yyyyy"));
144-
config.put("timezone", "invalid_timezone");
145-
try {
146-
factory.create(null, null, config);
147-
fail("invalid timezone should fail");
148-
} catch (IllegalArgumentException e) {
149-
assertThat(e.getMessage(), equalTo("The datetime zone id 'invalid_timezone' is not recognised"));
150-
}
111+
assertThat(processor.getTimezone().newInstance(Collections.emptyMap()).execute(), equalTo(timezone.getID()));
151112
}
152113

153114
public void testParseMatchFormats() throws Exception {
154-
DateProcessor.Factory factory = new DateProcessor.Factory();
155115
Map<String, Object> config = new HashMap<>();
156116
String sourceField = randomAlphaOfLengthBetween(1, 10);
157117
config.put("field", sourceField);
@@ -162,7 +122,6 @@ public void testParseMatchFormats() throws Exception {
162122
}
163123

164124
public void testParseMatchFormatsFailure() throws Exception {
165-
DateProcessor.Factory factory = new DateProcessor.Factory();
166125
Map<String, Object> config = new HashMap<>();
167126
String sourceField = randomAlphaOfLengthBetween(1, 10);
168127
config.put("field", sourceField);
@@ -177,7 +136,6 @@ public void testParseMatchFormatsFailure() throws Exception {
177136
}
178137

179138
public void testParseTargetField() throws Exception {
180-
DateProcessor.Factory factory = new DateProcessor.Factory();
181139
Map<String, Object> config = new HashMap<>();
182140
String sourceField = randomAlphaOfLengthBetween(1, 10);
183141
String targetField = randomAlphaOfLengthBetween(1, 10);

0 commit comments

Comments
 (0)