ingest: date_index_name processor template resolution

jakelandis · jakelandis · commit f9a6ec1e18c0 · 2018-07-10T12:48:18.000-05:00
Review changes:
* Change to TemplateScript.Factory instead of ValueSource.wrap
* Add support for templated index_name_format
* Remove unecessary changes
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
@@ -32,8 +32,8 @@
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.ingest.ValueSource;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.TemplateScript;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -44,23 +44,22 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
     public static final String TYPE = "date_index_name";
 
     private final String field;
-    private final String indexNamePrefix;
-    private final String dateRounding;
-    private final String indexNameFormat;
+    private final TemplateScript.Factory indexNamePrefixTemplate;
+    private final TemplateScript.Factory dateRoundingTemplate;
+    private final TemplateScript.Factory indexNameFormatTemplate;
     private final DateTimeZone timezone;
     private final List<Function<String, DateTime>> dateFormats;
-    private final ScriptService scriptService;
 
     DateIndexNameProcessor(String tag, String field, List<Function<String, DateTime>> dateFormats, DateTimeZone timezone,
-                           String indexNamePrefix, String dateRounding, String indexNameFormat, ScriptService scriptService) {
+                           TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate,
+                           TemplateScript.Factory indexNameFormatTemplate) {
         super(tag);
         this.field = field;
         this.timezone = timezone;
         this.dateFormats = dateFormats;
-        this.indexNamePrefix = indexNamePrefix;
-        this.dateRounding = dateRounding;
-        this.indexNameFormat = indexNameFormat;
-        this.scriptService = scriptService;
+        this.indexNamePrefixTemplate = indexNamePrefixTemplate;
+        this.dateRoundingTemplate = dateRoundingTemplate;
+        this.indexNameFormatTemplate = indexNameFormatTemplate;
     }
 
     @Override
@@ -87,6 +86,9 @@ public void execute(IngestDocument ingestDocument) throws Exception {
         if (dateTime == null) {
             throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException);
         }
+        String indexNamePrefix = ingestDocument.renderTemplate(indexNamePrefixTemplate);
+        String indexNameFormat = ingestDocument.renderTemplate(indexNameFormatTemplate);
+        String dateRounding = ingestDocument.renderTemplate(dateRoundingTemplate);
 
         DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat);
         StringBuilder builder = new StringBuilder()
@@ -98,7 +100,7 @@ public void execute(IngestDocument ingestDocument) throws Exception {
                     .append('}')
                 .append('>');
         String dynamicIndexName  = builder.toString();
-        ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), ValueSource.wrap(dynamicIndexName, scriptService));
+        ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), dynamicIndexName);
     }
 
     @Override
@@ -110,16 +112,16 @@ String getField() {
         return field;
     }
 
-    String getIndexNamePrefix() {
-        return indexNamePrefix;
+    TemplateScript.Factory getIndexNamePrefixTemplate() {
+        return indexNamePrefixTemplate;
     }
 
-    String getDateRounding() {
-        return dateRounding;
+    TemplateScript.Factory getDateRoundingTemplate() {
+        return dateRoundingTemplate;
     }
 
-    String getIndexNameFormat() {
-        return indexNameFormat;
+    TemplateScript.Factory getIndexNameFormatTemplate() {
+        return indexNameFormatTemplate;
     }
 
     DateTimeZone getTimezone() {
@@ -164,10 +166,16 @@ public DateIndexNameProcessor create(Map<String, Processor.Factory> registry, St
 
             String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
             String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", "");
+            TemplateScript.Factory indexNamePrefixTemplate =
+                ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_prefix", indexNamePrefix, scriptService);
             String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding");
+            TemplateScript.Factory dateRoundingTemplate =
+                ConfigurationUtils.compileTemplate(TYPE, tag, "date_rounding", dateRounding, scriptService);
             String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd");
-            return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix,
-                dateRounding, indexNameFormat, scriptService);
+            TemplateScript.Factory indexNameFormatTemplate =
+                ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService);
+            return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate,
+                dateRoundingTemplate, indexNameFormatTemplate);
         }
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
@@ -26,6 +26,7 @@
 import org.joda.time.DateTimeZone;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -40,9 +41,9 @@ public void testDefaults() throws Exception {
         DateIndexNameProcessor processor = factory.create(null, null, config);
         assertThat(processor.getDateFormats().size(), Matchers.equalTo(1));
         assertThat(processor.getField(), Matchers.equalTo("_field"));
-        assertThat(processor.getIndexNamePrefix(), Matchers.equalTo(""));
-        assertThat(processor.getDateRounding(), Matchers.equalTo("y"));
-        assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyy-MM-dd"));
+        assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo(""));
+        assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y"));
+        assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd"));
         assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC));
     }
 
@@ -64,7 +65,7 @@ public void testSpecifyOptionalSettings() throws Exception {
         config.put("index_name_format", "yyyyMMdd");
 
         processor = factory.create(null, null, config);
-        assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd"));
+        assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd"));
 
         config = new HashMap<>();
         config.put("field", "_field");
@@ -81,7 +82,7 @@ public void testSpecifyOptionalSettings() throws Exception {
         config.put("date_rounding", "y");
 
         processor = factory.create(null, null, config);
-        assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix"));
+        assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix"));
     }
 
     public void testRequiredFields() throws Exception {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
@@ -20,35 +20,24 @@
 
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.TestTemplateService;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
-import org.mockito.Matchers;
-import org.mockito.Mockito;
+import org.joda.time.format.DateTimeFormat;
 
 import java.util.Collections;
+import java.util.List;
 import java.util.Locale;
 import java.util.function.Function;
 
-import static org.elasticsearch.script.Script.DEFAULT_TEMPLATE_LANG;
 import static org.hamcrest.CoreMatchers.equalTo;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class DateIndexNameProcessorTests extends ESTestCase {
 
     public void testJodaPattern() throws Exception {
         Function<String, DateTime> function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
-        DateIndexNameProcessor processor = new DateIndexNameProcessor(
-                "_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC,
-                "events-", "y", "yyyyMMdd", TestTemplateService.instance()
-        );
-
+        DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function),
+            DateTimeZone.UTC, "events-", "y", "yyyyMMdd");
         IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
                 Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z"));
         processor.execute(document);
@@ -57,8 +46,8 @@ public void testJodaPattern() throws Exception {
 
     public void testTAI64N()throws Exception {
         Function<String, DateTime> function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null);
-        DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
-                DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance());
+        DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
+                DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
         IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
                 Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024"));
         dateProcessor.execute(document);
@@ -67,8 +56,8 @@ public void testTAI64N()throws Exception {
 
     public void testUnixMs()throws Exception {
         Function<String, DateTime> function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null);
-        DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
-                DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance());
+        DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
+                DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
         IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
                 Collections.singletonMap("_field", "1000500"));
         dateProcessor.execute(document);
@@ -82,29 +71,41 @@ public void testUnixMs()throws Exception {
 
     public void testUnix()throws Exception {
         Function<String, DateTime> function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
-        DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
-                DateTimeZone.UTC, "events-", "m", "yyyyMMdd", TestTemplateService.instance());
+        DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function),
+                DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
         IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
                 Collections.singletonMap("_field", "1000.5"));
         dateProcessor.execute(document);
         assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
     }
 
-    public void testTemplateSupported() throws Exception {
-        ScriptService scriptService = mock(ScriptService.class);
-        TestTemplateService.MockTemplateScript.Factory factory = new TestTemplateService.MockTemplateScript.Factory("script_result");
-        when(scriptService.compile(any(Script.class), Matchers.<ScriptContext<TemplateScript.Factory>>any())).thenReturn(factory);
-        when(scriptService.isLangSupported(DEFAULT_TEMPLATE_LANG)).thenReturn(true);
+    public void testTemplatedFields() throws Exception {
+        String indexNamePrefix = randomAlphaOfLength(10);
+        String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s");
+        String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy");
+        String date = Integer.toString(randomInt());
+        Function<String, DateTime> dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
+
+        DateIndexNameProcessor dateProcessor = createProcessor("_field",
+            Collections.singletonList(dateTimeFunction),  DateTimeZone.UTC, indexNamePrefix,
+            dateRounding, indexNameFormat);
 
-        DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field",
-            Collections.singletonList(DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null)),
-            DateTimeZone.UTC, "events-", "m", "yyyyMMdd", scriptService);
         IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null,
-            Collections.singletonMap("_field", "1000.5"));
+            Collections.singletonMap("_field", date));
         dateProcessor.execute(document);
 
-        // here we only care that the script was compiled and that it returned what we expect.
-        Mockito.verify(scriptService).compile(any(Script.class), Matchers.<ScriptContext<TemplateScript.Factory>>any());
-        assertThat(document.getSourceAndMetadata().get("_index"), equalTo("script_result"));
+        assertThat(document.getSourceAndMetadata().get("_index"),
+            equalTo("<"+indexNamePrefix+"{"+DateTimeFormat.forPattern(indexNameFormat)
+                .print(dateTimeFunction.apply(date))+"||/"+dateRounding+"{"+indexNameFormat+"|UTC}}>"));
+    }
+
+    private DateIndexNameProcessor createProcessor(String field, List<Function<String, DateTime>> dateFormats,
+                                                   DateTimeZone timezone, String indexNamePrefix, String dateRounding,
+                                                   String indexNameFormat) {
+        return new DateIndexNameProcessor(randomAlphaOfLength(10), field, dateFormats, timezone,
+            new TestTemplateService.MockTemplateScript.Factory(indexNamePrefix),
+            new TestTemplateService.MockTemplateScript.Factory(dateRounding),
+            new TestTemplateService.MockTemplateScript.Factory(indexNameFormat)
+        );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -406,21 +406,6 @@ public void setFieldValue(String path, Object value) {
         setFieldValue(path, value, false);
     }
 
-    /**
-     * Sets the provided value to the provided path in the document.
-     * Any non existing path element will be created.
-     * If the last item in the path is a list, the value will replace the existing list as a whole.
-     * Use {@link #appendFieldValue(String, Object)} to append values to lists instead.
-     * @param path The path within the document in dot-notation
-     * @param valueSource The value source that will produce the value or values to append to the existing ones
-     * @throws IllegalArgumentException if the path is null, empty, invalid or if the value cannot be set to the
-     * item identified by the provided path.
-     */
-    public void setFieldValue(String path, ValueSource valueSource) {
-        Map<String, Object> model = valueSource == null ? null : createTemplateModel();
-        setFieldValue(path, valueSource == null ? null : valueSource.copyAndResolve(model), false);
-    }
-
     /**
      * Sets the provided value to the provided path in the document.
      * Any non existing path element will be created. If the last element is a list,
