From 8e5cd0a391937fa6ec347811d834f25f41ea40c8 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Sat, 6 Feb 2021 17:39:47 -0600 Subject: [PATCH 1/3] media_type option for append processor --- .../ingest/common/AppendProcessor.java | 14 +++++++---- .../common/AppendProcessorFactoryTests.java | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java index fcadde8b47f3d..557cf214cc453 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java @@ -13,6 +13,7 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.ValueSource; +import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.TemplateScript; @@ -71,10 +72,15 @@ public AppendProcessor create(Map registry, String pr String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value"); boolean allowDuplicates = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "allow_duplicates", true); - TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, - "field", field, scriptService); - return new AppendProcessor(processorTag, description, compiledTemplate, ValueSource.wrap(value, scriptService), - allowDuplicates); + TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService); + String mediaType = ConfigurationUtils.readMediaTypeProperty(TYPE, processorTag, config, "media_type", "application/json"); + return new AppendProcessor( + processorTag, + description, + compiledTemplate, + ValueSource.wrap(value, scriptService, Map.of(Script.CONTENT_TYPE_OPTION, mediaType)), + allowDuplicates + ); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java index f351165503152..060aad95971c8 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -20,6 +21,7 @@ import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.containsString; public class AppendProcessorFactoryTests extends ESTestCase { @@ -92,4 +94,26 @@ public void testInvalidMustacheTemplate() throws Exception { assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script")); assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag)); } + + public void testMediaType() throws Exception { + // valid media type + String expectedMediaType = randomFrom(ConfigurationUtils.VALID_MEDIA_TYPES); + Map config = new HashMap<>(); + config.put("field", "field1"); + config.put("value", "value1"); + config.put("media_type", expectedMediaType); + String processorTag = randomAlphaOfLength(10); + AppendProcessor appendProcessor = factory.create(null, processorTag, null, config); + assertThat(appendProcessor.getTag(), equalTo(processorTag)); + + // invalid media type + expectedMediaType = randomValueOtherThanMany(m -> Arrays.asList(ConfigurationUtils.VALID_MEDIA_TYPES).contains(m), + () -> randomAlphaOfLengthBetween(5, 9)); + final Map config2 = new HashMap<>(); + config2.put("field", "field1"); + config2.put("value", "value1"); + config2.put("media_type", expectedMediaType); + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config2)); + assertThat(e.getMessage(), containsString("property does not contain a supported media type [" + expectedMediaType + "]")); + } } From cd16bcdf0e3d2370c0678d4af73867dd107a54f5 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 9 Aug 2021 12:22:28 -0500 Subject: [PATCH 2/3] include docs update --- docs/reference/ingest/processors/append.asciidoc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/reference/ingest/processors/append.asciidoc b/docs/reference/ingest/processors/append.asciidoc index 919cf92ec2ec6..86f3558518ac5 100644 --- a/docs/reference/ingest/processors/append.asciidoc +++ b/docs/reference/ingest/processors/append.asciidoc @@ -15,10 +15,13 @@ Accepts a single value or an array of values. [options="header"] |====== | Name | Required | Default | Description -| `field` | yes | - | The field to be appended to. Supports <>. -| `value` | yes | - | The value to be appended. Supports <>. +| `field` | yes | - | The field to be appended to. Supports <>. +| `value` | yes | - | The value to be appended. Supports <>. | `allow_duplicates` | no | true | If `false`, the processor does not append values already present in the field. +| `media_type` | no | `application/json` | The media type for encoding `value`. Applies only when `value` is a +<>. Must be one of `application/json`, `text/plain`, or +`application/x-www-form-urlencoded`. include::common-options.asciidoc[] |====== From 10764140d6b43266d3a5e869727cfc62ec637418 Mon Sep 17 00:00:00 2001 From: Dan Hermann Date: Mon, 9 Aug 2021 12:32:22 -0500 Subject: [PATCH 3/3] fix docs --- docs/reference/ingest/processors/append.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ingest/processors/append.asciidoc b/docs/reference/ingest/processors/append.asciidoc index 86f3558518ac5..c2d082c59c85c 100644 --- a/docs/reference/ingest/processors/append.asciidoc +++ b/docs/reference/ingest/processors/append.asciidoc @@ -20,7 +20,7 @@ Accepts a single value or an array of values. | `allow_duplicates` | no | true | If `false`, the processor does not append values already present in the field. | `media_type` | no | `application/json` | The media type for encoding `value`. Applies only when `value` is a -<>. Must be one of `application/json`, `text/plain`, or +<>. Must be one of `application/json`, `text/plain`, or `application/x-www-form-urlencoded`. include::common-options.asciidoc[] |======