Script: Add Metadata to ingest context.

docs/changelog/87309.yaml

@@ -0,0 +1,5 @@
+pr: 87309
+summary: "Script: Add Metadata to ingest context"
+area: Infra/Scripting
+type: enhancement
+issues: []

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java

@@ -40,26 +40,26 @@ public final class ScriptProcessor extends AbstractProcessor {
 
     private final Script script;
     private final ScriptService scriptService;
-    private final IngestScript precompiledIngestScript;
+    private final IngestScript.Factory precompiledIngestScriptFactory;
 
     /**
      * Processor that evaluates a script with an ingest document in its context
      *  @param tag The processor's tag.
      * @param description The processor's description.
      * @param script The {@link Script} to execute.
-     * @param precompiledIngestScript The {@link Script} precompiled
+     * @param precompiledIngestScriptFactory The {@link Script} precompiled script
      * @param scriptService The {@link ScriptService} used to execute the script.
      */
     ScriptProcessor(
         String tag,
         String description,
         Script script,
-        @Nullable IngestScript precompiledIngestScript,
+        @Nullable IngestScript.Factory precompiledIngestScriptFactory,
         ScriptService scriptService
     ) {
         super(tag, description);
         this.script = script;
-        this.precompiledIngestScript = precompiledIngestScript;
+        this.precompiledIngestScriptFactory = precompiledIngestScriptFactory;
         this.scriptService = scriptService;
     }
 
@@ -71,14 +71,11 @@ public final class ScriptProcessor extends AbstractProcessor {
     @Override
     public IngestDocument execute(IngestDocument document) {
         document.doNoSelfReferencesCheck(true);
-        final IngestScript ingestScript;
-        if (precompiledIngestScript == null) {
-            IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
-            ingestScript = factory.newInstance(script.getParams());
-        } else {
-            ingestScript = precompiledIngestScript;
+        IngestScript.Factory factory = precompiledIngestScriptFactory;
+        if (factory == null) {
+            factory = scriptService.compile(script, IngestScript.CONTEXT);
         }
-        ingestScript.execute(document.getSourceAndMetadata());
+        factory.newInstance(script.getParams(), document.getMetadata(), document.getSourceAndMetadata()).execute();
         return document;
     }
 
@@ -91,8 +88,8 @@ Script getScript() {
         return script;
     }
 
-    IngestScript getPrecompiledIngestScript() {
-        return precompiledIngestScript;
+    IngestScript.Factory getPrecompiledIngestScriptFactory() {
+        return precompiledIngestScriptFactory;
     }
 
     public static final class Factory implements Processor.Factory {
@@ -120,16 +117,17 @@ public ScriptProcessor create(
                 Arrays.asList("id", "source", "inline", "lang", "params", "options").forEach(config::remove);
 
                 // verify script is able to be compiled before successfully creating processor.
-                IngestScript ingestScript = null;
+                IngestScript.Factory ingestScriptFactory = null;
                 try {
-                    final IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT);
-                    if (ScriptType.INLINE.equals(script.getType())) {
-                        ingestScript = factory.newInstance(script.getParams());
+                    ingestScriptFactory = scriptService.compile(script, IngestScript.CONTEXT);
+                    if (ScriptType.STORED.equals(script.getType())) {
+                        // do not cache stored scripts lest they change and invalidate the cached value
+                        ingestScriptFactory = null;
                     }
                 } catch (ScriptException e) {
                     throw newConfigurationException(TYPE, processorTag, null, e);
                 }
-                return new ScriptProcessor(processorTag, description, script, ingestScript, scriptService);
+                return new ScriptProcessor(processorTag, description, script, ingestScriptFactory, scriptService);
             }
         }
     }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java

@@ -48,7 +48,7 @@ public void testEscapeFields() throws Exception {
         processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
         processor.execute(document);
         assertThat(document.getSource().size(), equalTo(1));
-        assertThat(document.getMetadata().size(), equalTo(1)); // the default version
+        assertThat(document.getMetadataMap().size(), equalTo(1)); // the default version
         assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2));
         assertThat(document.getFieldValue("foo.bar.0", String.class), equalTo("baz2"));
         assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("baz1"));
@@ -60,7 +60,7 @@ public void testEscapeFields() throws Exception {
         processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
         processor.execute(document);
         assertThat(document.getSource().size(), equalTo(1));
-        assertThat(document.getMetadata().size(), equalTo(1)); // the default version
+        assertThat(document.getMetadataMap().size(), equalTo(1)); // the default version
         assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2));
         assertThat(document.getFieldValue("foo.bar.0", Integer.class), equalTo(1));
         assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("2"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java

@@ -10,6 +10,8 @@
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.TestIngestDocument;
 import org.elasticsearch.script.IngestScript;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
@@ -155,9 +157,10 @@ public void testInlineIsCompiled() throws Exception {
         assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
         assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE));
         assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
-        assertNotNull(processor.getPrecompiledIngestScript());
-        Map<String, Object> ctx = new HashMap<>();
-        processor.getPrecompiledIngestScript().execute(ctx);
+        assertNotNull(processor.getPrecompiledIngestScriptFactory());
+        IngestDocument doc = TestIngestDocument.emptyIngestDocument();
+        Map<String, Object> ctx = TestIngestDocument.emptyIngestDocument().getIngestSourceAndMetadata();
+        processor.getPrecompiledIngestScriptFactory().newInstance(null, doc.getMetadata(), ctx).execute();
         assertThat(ctx.get("foo"), equalTo("bar"));
     }
 
@@ -171,6 +174,6 @@ public void testStoredIsNotCompiled() throws Exception {
         assertNull(processor.getScript().getLang());
         assertThat(processor.getScript().getType(), equalTo(ScriptType.STORED));
         assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap()));
-        assertNull(processor.getPrecompiledIngestScript());
+        assertNull(processor.getPrecompiledIngestScriptFactory());
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java

@@ -32,7 +32,7 @@ public class ScriptProcessorTests extends ESTestCase {
 
     private ScriptService scriptService;
     private Script script;
-    private IngestScript ingestScript;
+    private IngestScript.Factory ingestScriptFactory;
 
     @Before
     public void setupScripting() {
@@ -53,7 +53,7 @@ public void setupScripting() {
             () -> 1L
         );
         script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap());
-        ingestScript = scriptService.compile(script, IngestScript.CONTEXT).newInstance(script.getParams());
+        ingestScriptFactory = scriptService.compile(script, IngestScript.CONTEXT);
     }
 
     public void testScriptingWithoutPrecompiledScriptFactory() throws Exception {
@@ -64,7 +64,7 @@ public void testScriptingWithoutPrecompiledScriptFactory() throws Exception {
     }
 
     public void testScriptingWithPrecompiledIngestScript() {
-        ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), null, script, ingestScript, scriptService);
+        ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), null, script, ingestScriptFactory, scriptService);
         IngestDocument ingestDocument = randomDocument();
         processor.execute(ingestDocument);
         assertIngestDocument(ingestDocument);

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml

@@ -314,3 +314,45 @@ teardown:
             }
           }
   - match: { error.root_cause.0.reason: "Failed to generate the source document for ingest pipeline [my_pipeline]" }
+
+---
+"Test metadata":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "metadata",
+            "processors": [
+              {
+                "script" : {
+                  "lang": "painless",
+                  "source" : "Metadata m = metadata(); m.index += '2'; m.id += 'extra'; m.routing = 'myRouting'; m.versionType = 'external'; m.version = 5"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test2
+        id: "1extra"
+        version: 4
+        version_type: "external"
+        body: {source_field: "foobar"}
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: {source_field: "bazqux"}
+
+  - do:
+      get:
+        index: test2
+        id: "1extra"
+  - match: { _source.source_field: "bazqux" }
+  - match: { _version: 5 }
+  - match: { _routing: "myRouting" }

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt

@@ -18,3 +18,26 @@ class org.elasticsearch.painless.api.Json {
   String dump(def)
   String dump(def, boolean)
 }
+
+class org.elasticsearch.script.Metadata {
+    String getIndex()
+    void setIndex(String)
+
+    String getId()
+    void setId(String)
+
+    String getRouting()
+    void setRouting(String)
+
+    long getVersion()
+    void setVersion(long)
+
+    String getVersionType()
+    void setVersionType(String)
+
+    ZonedDateTime getTimestamp()
+}
+
+class org.elasticsearch.script.IngestScript {
+    Metadata metadata()
+}

server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java

@@ -115,7 +115,7 @@ public void testSimulate() throws Exception {
         source.put("processed", true);
         IngestDocument ingestDocument = new IngestDocument("index", "id", Versions.MATCH_ANY, null, null, source);
         assertThat(simulateDocumentBaseResult.getIngestDocument().getSource(), equalTo(ingestDocument.getSource()));
-        assertThat(simulateDocumentBaseResult.getIngestDocument().getMetadata(), equalTo(ingestDocument.getMetadata()));
+        assertThat(simulateDocumentBaseResult.getIngestDocument().getMetadataMap(), equalTo(ingestDocument.getMetadataMap()));
         assertThat(simulateDocumentBaseResult.getFailure(), nullValue());
 
         // cleanup

server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java

@@ -42,7 +42,9 @@ final class WriteableIngestDocument implements Writeable, ToXContentFragment {
         "ingest_document",
         true,
         a -> {
-            Map<String, Object> sourceAndMetadata = Maps.newHashMapWithExpectedSize(5);
+            Map<String, Object> source = (Map<String, Object>) a[5];
+            Map<String, Object> sourceAndMetadata = Maps.newHashMapWithExpectedSize(5 + source.size());
+            sourceAndMetadata.putAll(source);
             sourceAndMetadata.put(Metadata.INDEX.getFieldName(), a[0]);
             sourceAndMetadata.put(Metadata.ID.getFieldName(), a[1]);
             if (a[2] != null) {
@@ -54,7 +56,6 @@ final class WriteableIngestDocument implements Writeable, ToXContentFragment {
             if (a[4] != null) {
                 sourceAndMetadata.put(Metadata.VERSION_TYPE.getFieldName(), a[4]);
             }
-            sourceAndMetadata.putAll((Map<String, Object>) a[5]);
             Map<String, Object> ingestMetadata = (Map<String, Object>) a[6];
             return new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, ingestMetadata));
         }
@@ -106,7 +107,7 @@ IngestDocument getIngestDocument() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(DOC_FIELD);
-        Map<String, Object> metadataMap = ingestDocument.getMetadata();
+        Map<String, Object> metadataMap = ingestDocument.getMetadataMap();
         for (Map.Entry<String, Object> metadata : metadataMap.entrySet()) {
             if (metadata.getValue() != null) {
                 builder.field(metadata.getKey(), metadata.getValue().toString());

server/src/main/java/org/elasticsearch/ingest/IngestDocument.java

@@ -733,17 +733,24 @@ public Map<String, Object> getSourceAndMetadata() {
     /**
      * Get source and metadata map as {@link IngestSourceAndMetadata}
      */
-    IngestSourceAndMetadata getIngestSourceAndMetadata() {
+    public IngestSourceAndMetadata getIngestSourceAndMetadata() {
         return sourceAndMetadata;
     }
 
     /**
      * Get all Metadata values in a Map
      */
-    public Map<String, Object> getMetadata() {
+    public Map<String, Object> getMetadataMap() {
         return sourceAndMetadata.getMetadata();
     }
 
+    /**
+     * Get the strongly typed metadata
+     */
+    public org.elasticsearch.script.Metadata getMetadata() {
+        return sourceAndMetadata;
+    }
+
     /**
      * Get all source values in a Map
      */

server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.script.Metadata;
 
 import java.time.ZonedDateTime;
 import java.util.AbstractCollection;
@@ -40,7 +41,7 @@
  *
  * The map is expected to be used by processors, server code should the typed getter and setters where possible.
  */
-class IngestSourceAndMetadata extends AbstractMap<String, Object> {
+class IngestSourceAndMetadata extends AbstractMap<String, Object> implements Metadata {
     protected final ZonedDateTime timestamp;
 
     /**

server/src/main/java/org/elasticsearch/script/IngestScript.java

@@ -18,7 +18,7 @@
  */
 public abstract class IngestScript {
 
-    public static final String[] PARAMETERS = { "ctx" };
+    public static final String[] PARAMETERS = {};
 
     /** The context used to compile {@link IngestScript} factories. */
     public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>(
@@ -33,18 +33,36 @@ public abstract class IngestScript {
     /** The generic runtime parameters for the script. */
     private final Map<String, Object> params;
 
-    public IngestScript(Map<String, Object> params) {
+    /** The metadata available to the script */
+    private final Metadata metadata;
+
+    /** The metadata and source available to the script */
+    private final Map<String, Object> ctx;
+
+    public IngestScript(Map<String, Object> params, Metadata metadata, Map<String, Object> ctx) {
         this.params = params;
+        this.metadata = metadata;
+        this.ctx = ctx;
     }
 
     /** Return the parameters for this script. */
     public Map<String, Object> getParams() {
         return params;
     }
 
-    public abstract void execute(Map<String, Object> ctx);
+    /** Provides backwards compatibility access to ctx */
+    public Map<String, Object> getCtx() {
+        return ctx;
+    }
+
+    /** Return the ingest metadata object */
+    public Metadata metadata() {
+        return metadata;
+    }
+
+    public abstract void execute();
 
     public interface Factory {
-        IngestScript newInstance(Map<String, Object> params);
+        IngestScript newInstance(Map<String, Object> params, Metadata metadata, Map<String, Object> ctx);
     }
 }