From 8396488c35f4e9451c39e3d35d81da2c48447679 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 1 Jun 2022 18:11:46 -0500 Subject: [PATCH 01/15] Script: Add Metadata to ingest context. Adds the `Metadata` class and `meta()` method to the ingest context. Metadata has getters and setters for index, id, routing, version and versionType. It also has a getter for timestamp. The `VersionType` enum is available for use with the versionType getter and setter. Refs: #86472 --- .../ingest/common/ScriptProcessor.java | 35 +++-- .../common/ScriptProcessorFactoryTests.java | 6 +- .../ingest/common/ScriptProcessorTests.java | 6 +- .../test/ingest/190_script_processor.yml | 42 ++++++ .../painless/PainlessPlugin.java | 14 +- .../org.elasticsearch.script.ingest.meta.txt | 34 +++++ .../elasticsearch/ingest/IngestDocument.java | 14 ++ .../elasticsearch/script/IngestScript.java | 82 +++++++++- .../script/field/MapBackedMetadata.java | 112 ++++++++++++++ .../script/IngestScriptMetadataTests.java | 36 +++++ .../script/field/MapBackedMetadataTests.java | 142 ++++++++++++++++++ .../script/MockScriptEngine.java | 5 +- .../ml/integration/MlNativeIntegTestCase.java | 4 +- .../xpack/ml/MlSingleNodeTestCase.java | 4 +- .../xpack/ml/support/BaseMlIntegTestCase.java | 4 +- 15 files changed, 497 insertions(+), 43 deletions(-) create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt create mode 100644 server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java create mode 100644 server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java create mode 100644 server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index c455bfdaced1a..00b7ea773b489 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -40,26 +40,26 @@ public final class ScriptProcessor extends AbstractProcessor { private final Script script; private final ScriptService scriptService; - private final IngestScript precompiledIngestScript; + private final IngestScript.Factory precompiledIngestScriptFactory; /** * Processor that evaluates a script with an ingest document in its context * @param tag The processor's tag. * @param description The processor's description. * @param script The {@link Script} to execute. - * @param precompiledIngestScript The {@link Script} precompiled + * @param precompiledIngestScriptFactory The {@link Script} precompiled script * @param scriptService The {@link ScriptService} used to execute the script. */ ScriptProcessor( String tag, String description, Script script, - @Nullable IngestScript precompiledIngestScript, + @Nullable IngestScript.Factory precompiledIngestScriptFactory, ScriptService scriptService ) { super(tag, description); this.script = script; - this.precompiledIngestScript = precompiledIngestScript; + this.precompiledIngestScriptFactory = precompiledIngestScriptFactory; this.scriptService = scriptService; } @@ -70,14 +70,12 @@ public final class ScriptProcessor extends AbstractProcessor { */ @Override public IngestDocument execute(IngestDocument document) { - final IngestScript ingestScript; - if (precompiledIngestScript == null) { - IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT); - ingestScript = factory.newInstance(script.getParams()); - } else { - ingestScript = precompiledIngestScript; + IngestScript.Factory factory = precompiledIngestScriptFactory; + if (factory == null) { + factory = scriptService.compile(script, IngestScript.CONTEXT); } - ingestScript.execute(document.getSourceAndMetadata()); + factory.newInstance(script.getParams(), new IngestScript.Metadata(document.getSourceAndMetadata(), document.getTimestamp())) + .execute(); return document; } @@ -90,8 +88,8 @@ Script getScript() { return script; } - IngestScript getPrecompiledIngestScript() { - return precompiledIngestScript; + IngestScript.Factory getPrecompiledIngestScriptFactory() { + return precompiledIngestScriptFactory; } public static final class Factory implements Processor.Factory { @@ -119,16 +117,17 @@ public ScriptProcessor create( Arrays.asList("id", "source", "inline", "lang", "params", "options").forEach(config::remove); // verify script is able to be compiled before successfully creating processor. - IngestScript ingestScript = null; + IngestScript.Factory ingestScriptFactory = null; try { - final IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT); - if (ScriptType.INLINE.equals(script.getType())) { - ingestScript = factory.newInstance(script.getParams()); + ingestScriptFactory = scriptService.compile(script, IngestScript.CONTEXT); + if (ScriptType.STORED.equals(script.getType())) { + // do not cache stored scripts lest they change and invalidate the cached value + ingestScriptFactory = null; } } catch (ScriptException e) { throw newConfigurationException(TYPE, processorTag, null, e); } - return new ScriptProcessor(processorTag, description, script, ingestScript, scriptService); + return new ScriptProcessor(processorTag, description, script, ingestScriptFactory, scriptService); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index 8042253b8b1df..897a7ab9afae2 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -155,9 +155,9 @@ public void testInlineIsCompiled() throws Exception { assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG)); assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE)); assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap())); - assertNotNull(processor.getPrecompiledIngestScript()); + assertNotNull(processor.getPrecompiledIngestScriptFactory()); Map ctx = new HashMap<>(); - processor.getPrecompiledIngestScript().execute(ctx); + processor.getPrecompiledIngestScriptFactory().newInstance(null, new IngestScript.Metadata(ctx, null)).execute(); assertThat(ctx.get("foo"), equalTo("bar")); } @@ -171,6 +171,6 @@ public void testStoredIsNotCompiled() throws Exception { assertNull(processor.getScript().getLang()); assertThat(processor.getScript().getType(), equalTo(ScriptType.STORED)); assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap())); - assertNull(processor.getPrecompiledIngestScript()); + assertNull(processor.getPrecompiledIngestScriptFactory()); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java index 6add04b9b9b61..972ca029b7b03 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java @@ -32,7 +32,7 @@ public class ScriptProcessorTests extends ESTestCase { private ScriptService scriptService; private Script script; - private IngestScript ingestScript; + private IngestScript.Factory ingestScriptFactory; @Before public void setupScripting() { @@ -53,7 +53,7 @@ public void setupScripting() { () -> 1L ); script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()); - ingestScript = scriptService.compile(script, IngestScript.CONTEXT).newInstance(script.getParams()); + ingestScriptFactory = scriptService.compile(script, IngestScript.CONTEXT); } public void testScriptingWithoutPrecompiledScriptFactory() throws Exception { @@ -64,7 +64,7 @@ public void testScriptingWithoutPrecompiledScriptFactory() throws Exception { } public void testScriptingWithPrecompiledIngestScript() { - ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), null, script, ingestScript, scriptService); + ScriptProcessor processor = new ScriptProcessor(randomAlphaOfLength(10), null, script, ingestScriptFactory, scriptService); IngestDocument ingestDocument = randomDocument(); processor.execute(ingestDocument); assertIngestDocument(ingestDocument); diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml index 4f0d84f52608d..1d00ea9b5e46d 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml @@ -314,3 +314,45 @@ teardown: } } - match: { error.root_cause.0.reason: "Iterable object is self-referencing itself (ingest pipeline [my_pipeline])" } + +--- +"Test metadata": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "metadata", + "processors": [ + { + "script" : { + "lang": "painless", + "source" : "Metadata m = meta(); m.index += '2'; m.id += 'extra'; m.routing = 'myRouting'; m.versionType = VersionType.EXTERNAL; m.version = Long.valueOf(5);" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test2 + id: "1extra" + version: 4 + version_type: "external" + body: {source_field: "foobar"} + + - do: + index: + index: test + id: "1" + pipeline: "my_pipeline" + body: {source_field: "bazqux"} + + - do: + get: + index: test2 + id: "1extra" + - match: { _source.source_field: "bazqux" } + - match: { _version: 5 } + - match: { _routing: "myRouting" } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java index 94e4a08084e39..7c0e90eb8ef6b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java @@ -90,13 +90,13 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens for (ScriptContext context : ScriptModule.CORE_CONTEXTS.values()) { List contextWhitelists = new ArrayList<>(); - if (PainlessPlugin.class.getResourceAsStream("org.elasticsearch.script." + context.name.replace('-', '_') + ".txt") != null) { - contextWhitelists.add( - WhitelistLoader.loadFromResourceFiles( - PainlessPlugin.class, - "org.elasticsearch.script." + context.name.replace('-', '_') + ".txt" - ) - ); + String contextWhitelistName = "org.elasticsearch.script." + context.name.replace('-', '_'); + if (PainlessPlugin.class.getResourceAsStream(contextWhitelistName + ".txt") != null) { + contextWhitelists.add(WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, contextWhitelistName + ".txt")); + if (PainlessPlugin.class.getResourceAsStream(contextWhitelistName + ".meta.txt") != null) { + Whitelist meta = WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, contextWhitelistName + ".meta.txt"); + contextWhitelists.add(meta); + } } whitelists.put(context, contextWhitelists); diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt new file mode 100644 index 0000000000000..f7ca27cd00e7a --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt @@ -0,0 +1,34 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +# This file only whitelists the script class and the Metadata class for ingest scripts. +# Due to name conflicts, this whitelist cannot be loaded with other scripts with conflicting Metadata types. + +class org.elasticsearch.script.IngestScript$Metadata @alias[class="Metadata"] { + String getIndex() + void setIndex(String) + String getId() + void setId(String) + String getRouting() + void setRouting(String) + Long getVersion() + void setVersion(Long) + VersionType getVersionType() + void setVersionType(VersionType) + ZonedDateTime getTimestamp() +} + +class org.elasticsearch.script.IngestScript { + Metadata meta() +} + +class org.elasticsearch.index.VersionType { + VersionType INTERNAL + VersionType EXTERNAL + VersionType EXTERNAL_GTE +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 9c1214196c4a0..9fdbecb3e3fc5 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -741,6 +741,20 @@ public Map getSourceAndMetadata() { return this.sourceAndMetadata; } + /** + * Fetch the timestamp from the ingestMetadata, if it exists + * @return the timestamp for the document or null + */ + public ZonedDateTime getTimestamp() { + if (ingestMetadata == null) { + return null; + } + if (ingestMetadata.get(TIMESTAMP)instanceof ZonedDateTime timestamp) { + return timestamp; + } + return null; + } + @SuppressWarnings("unchecked") public static Map deepCopyMap(Map source) { return (Map) deepCopy(source); diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java index 705366ff4bf51..089a3859b8021 100644 --- a/server/src/main/java/org/elasticsearch/script/IngestScript.java +++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java @@ -10,7 +10,11 @@ package org.elasticsearch.script; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.script.field.MapBackedMetadata; +import java.time.ZonedDateTime; +import java.util.Locale; import java.util.Map; /** @@ -18,7 +22,7 @@ */ public abstract class IngestScript { - public static final String[] PARAMETERS = { "ctx" }; + public static final String[] PARAMETERS = {}; /** The context used to compile {@link IngestScript} factories. */ public static final ScriptContext CONTEXT = new ScriptContext<>( @@ -33,8 +37,11 @@ public abstract class IngestScript { /** The generic runtime parameters for the script. */ private final Map params; - public IngestScript(Map params) { + private final Metadata metadata; + + public IngestScript(Map params, Metadata metadata) { this.params = params; + this.metadata = metadata; } /** Return the parameters for this script. */ @@ -42,9 +49,76 @@ public Map getParams() { return params; } - public abstract void execute(Map ctx); + public Map getCtx() { + return metadata != null ? metadata.store.getMap() : null; + } + + public Metadata meta() { + return metadata; + } + + public abstract void execute(); public interface Factory { - IngestScript newInstance(Map params); + IngestScript newInstance(Map params, Metadata metadata); + } + + /** + * Metadata available to scripts, backed by the ctx map. + */ + public static class Metadata { + private final MapBackedMetadata store; + private final ZonedDateTime timestamp; + public static final String VERSION_TYPE = "_version_type"; + + public Metadata(Map ctx, ZonedDateTime timestamp) { + store = new MapBackedMetadata(ctx); + this.timestamp = timestamp; + } + + public String getIndex() { + return store.getIndex(); + } + + public void setIndex(String index) { + store.setIndex(index); + } + + public String getId() { + return store.getId(); + } + + public void setId(String id) { + store.setId(id); + } + + public String getRouting() { + return store.getRouting(); + } + + public void setRouting(String routing) { + store.setRouting(routing); + } + + public Long getVersion() { + return store.getVersion(); + } + + public void setVersion(Long version) { + store.setVersion(version); + } + + public VersionType getVersionType() { + String str = store.getString(VERSION_TYPE); + return str != null ? VersionType.fromString(str.toLowerCase(Locale.ROOT)) : null; + } + + public void setVersionType(VersionType versionType) { + store.set(VERSION_TYPE, versionType != null ? VersionType.toString(versionType) : null); + } + + public ZonedDateTime getTimestamp() { + return timestamp; + } } } diff --git a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java new file mode 100644 index 0000000000000..a1dd670836262 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.elasticsearch.common.util.Maps; + +import java.util.Map; +import java.util.Objects; + +/** Metadata storage backed by a Map for compatibility with the ctx Map used by ingest and update scripts. */ +public class MapBackedMetadata { + public static final String INDEX = "_index"; + public static final String ID = "_id"; + public static final String VERSION = "_version"; + public static final String ROUTING = "_routing"; + public static final String SOURCE = "_source"; + + protected Map map; + + public MapBackedMetadata(Map map) { + this.map = map; + } + + public MapBackedMetadata(int size) { + this.map = Maps.newMapWithExpectedSize(size); + } + + public String getIndex() { + return getString(INDEX); + } + + public MapBackedMetadata setIndex(String index) { + return set(INDEX, index); + } + + public String getId() { + return getString(ID); + } + + public MapBackedMetadata setId(String id) { + return set(ID, id); + } + + public Long getVersion() { + Object obj = getRawVersion(); + if (obj == null) { + return null; + } else if (obj instanceof Number number) { + long version = number.longValue(); + if (number.doubleValue() != version) { + // did we round? + throw new IllegalArgumentException( + "version may only be set to an int or a long but was [" + number + "] with type [" + obj.getClass().getName() + "]" + ); + } + return version; + } + throw new IllegalArgumentException( + "version may only be set to an int or a long but was [" + obj + "] with type [" + obj.getClass().getName() + "]" + ); + } + + protected Object getRawVersion() { + return map.get(VERSION); + } + + public MapBackedMetadata setVersion(Long version) { + return set(VERSION, version); + } + + public String getRouting() { + return getString(ROUTING); + } + + public MapBackedMetadata setRouting(String routing) { + return set(ROUTING, routing); + } + + public String getString(String key) { + return Objects.toString(map.get(key), null); + } + + public MapBackedMetadata set(String key, Object value) { + map.put(key, value); + return this; + } + + public Map getMap() { + return map; + } + + public MapBackedMetadata setSource(Map source) { + map.put(SOURCE, source); + return this; + } + + @SuppressWarnings("unchecked") + public Map getSource() { + Object source = map.get(SOURCE); + if (source instanceof Map map) { + return (Map) map; + } else { + throw new IllegalArgumentException("source should be a map, not [" + source + "] with [" + source.getClass().getName() + "]"); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java new file mode 100644 index 0000000000000..436bc41ce3dbe --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.index.VersionType; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class IngestScriptMetadataTests extends ESTestCase { + public void testVersionType() { + Map ctx = new HashMap<>(); + IngestScript.Metadata m = new IngestScript.Metadata(ctx, null); + assertNull(m.getVersionType()); + for (VersionType vt : VersionType.values()) { + m.setVersionType(vt); + assertEquals(VersionType.toString(vt), ctx.get("_version_type")); + } + + + for (VersionType vt : VersionType.values()) { + ctx.put("_version_type", VersionType.toString(vt)); + assertEquals(vt, m.getVersionType()); + } + + m.setVersionType(null); + assertNull(m.getVersionType()); + } +} diff --git a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java new file mode 100644 index 0000000000000..c1f4adc36b453 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class MapBackedMetadataTests extends ESTestCase { + public void testString() { + MapBackedMetadata m = new MapBackedMetadata(1); + String key = "myKey"; + String value = "myValue"; + assertNull(m.getString(key)); + assertNull(m.getMap().get(key)); + + m.set(key, value); + assertEquals(value, m.getString(key)); + assertEquals(value, m.getMap().get(key)); + + m.set(key, 1); + assertEquals("1", m.getString(key)); + assertEquals(1, m.getMap().get(key)); + + m.set(key, null); + assertNull(m.getString(key)); + assertNull(m.getMap().get(key)); + + m.set(key, value); + assertEquals(value, m.getString(key)); + assertEquals(value, m.getMap().get(key)); + + m.getMap().remove(key); + assertNull(m.getString(key)); + assertNull(m.getMap().get(key)); + } + + public void testIndex() { + String index = "myIndex"; + MapBackedMetadata m = new MapBackedMetadata(1); + assertNull(m.getIndex()); + assertNull(m.getMap().get("_index")); + + m.setIndex(index); + assertEquals(index, m.getIndex()); + assertEquals(index, m.getMap().get("_index")); + } + + public void testId() { + String id = "myId"; + MapBackedMetadata m = new MapBackedMetadata(1); + assertNull(m.getId()); + assertNull(m.getMap().get("_id")); + + m.setId(id); + assertEquals(id, m.getId()); + assertEquals(id, m.getMap().get("_id")); + } + + public void testRouting() { + String routing = "myRouting"; + MapBackedMetadata m = new MapBackedMetadata(1); + assertNull(m.getRouting()); + assertNull(m.getMap().get("_routing")); + + m.setRouting(routing); + assertEquals(routing, m.getRouting()); + assertEquals(routing, m.getMap().get("_routing")); + } + + public void testVersion() { + long version = 500; + MapBackedMetadata m = new MapBackedMetadata(1); + assertNull(m.getVersion()); + assertNull(m.getMap().get("_version")); + + m.setVersion(version); + assertEquals(Long.valueOf(version), m.getVersion()); + assertEquals(version, m.getRawVersion()); + assertEquals(version, m.getMap().get("_version")); + + String badVersion = "badVersion"; + m.set("_version", badVersion); + assertEquals(badVersion, m.getRawVersion()); + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, m::getVersion); + assertEquals("version may only be set to an int or a long but was [badVersion] with type [java.lang.String]", err.getMessage()); + + double tooBig = Double.MAX_VALUE; + m.set("_version", tooBig); + assertEquals(tooBig, m.getRawVersion()); + err = expectThrows(IllegalArgumentException.class, m::getVersion); + assertEquals("version may only be set to an int or a long but was [" + tooBig + "] with type [java.lang.Double]", err.getMessage()); + + long justRight = (long) 1 << 52; + m.set("_version", (double) justRight); + assertEquals(Long.valueOf(justRight), m.getVersion()); + assertEquals((double) justRight, m.getRawVersion()); + } + + public void testSource() { + MapBackedMetadata m = new MapBackedMetadata(1); + Map source = new HashMap<>(); + source.put("foo", "bar"); + + m.setSource(source); + assertEquals(source, m.getSource()); + assertEquals(source, m.getMap().get("_source")); + + source.put("baz", "qux"); + assertEquals("qux", m.getSource().get("baz")); + + m.getMap().put("_source", "mySource"); + + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, m::getSource); + assertEquals("source should be a map, not [mySource] with [java.lang.String]", err.getMessage()); + } + + public void testCtx() { + Map ctx = new HashMap<>(); + ctx.put("_index", "myIndex"); + ctx.put("_id", "myId"); + ctx.put("_version", 200); + ctx.put("_routing", "myRouting"); + Map source = new HashMap<>(); + source.put("foo", "bar"); + ctx.put("_source", source); + + MapBackedMetadata m = new MapBackedMetadata(ctx); + assertEquals("myIndex", m.getIndex()); + assertEquals("myId", m.getId()); + assertEquals(Long.valueOf(200), m.getVersion()); + assertEquals("myRouting", m.getRouting()); + assertEquals("bar", m.getSource().get("foo")); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index 42e30159c3d91..14e4229722ff2 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -148,9 +148,10 @@ public boolean needs_score() { } else if (context.instanceClazz.equals(BytesRefSortScript.class)) { return context.factoryClazz.cast(new MockBytesRefSortScriptFactory(script)); } else if (context.instanceClazz.equals(IngestScript.class)) { - IngestScript.Factory factory = vars -> new IngestScript(vars) { + IngestScript.Factory factory = (parameters, metadata) -> new IngestScript(parameters, metadata) { @Override - public void execute(Map ctx) { + public void execute() { + final Map ctx = getCtx(); script.apply(ctx); } }; diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index c29ecc430726b..ec0d0f3453f02 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -440,9 +440,9 @@ public T compile(String name, String script, ScriptContext context, Map 0.0))); } if (context.name.equals("ingest")) { - IngestScript.Factory factory = vars -> new IngestScript(vars) { + IngestScript.Factory factory = (params, metadata) -> new IngestScript(params, metadata) { @Override - public void execute(Map ctx) {} + public void execute() {} }; return context.factoryClazz.cast(factory); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java index d19a88043cef9..5a06203e74a05 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java @@ -200,9 +200,9 @@ public T compile(String name, String script, ScriptContext context, Map 0.0))); } if (context.name.equals("ingest")) { - IngestScript.Factory factory = vars -> new IngestScript(vars) { + IngestScript.Factory factory = (vars, metadata) -> new IngestScript(vars, metadata) { @Override - public void execute(Map ctx) {} + public void execute() {} }; return context.factoryClazz.cast(factory); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index 5cbe7aba686c1..d3f4fa1169542 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -553,9 +553,9 @@ public T compile(String name, String script, ScriptContext context, Map 0.0))); } if (context.name.equals("ingest")) { - IngestScript.Factory factory = vars -> new IngestScript(vars) { + IngestScript.Factory factory = (vars, metadata) -> new IngestScript(vars, metadata) { @Override - public void execute(Map ctx) {} + public void execute() {} }; return context.factoryClazz.cast(factory); } From 7393951696f9f65a426a2973b19f709e3537613e Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 1 Jun 2022 18:21:00 -0500 Subject: [PATCH 02/15] Update docs/changelog/87309.yaml --- docs/changelog/87309.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/87309.yaml diff --git a/docs/changelog/87309.yaml b/docs/changelog/87309.yaml new file mode 100644 index 0000000000000..61c1b28386702 --- /dev/null +++ b/docs/changelog/87309.yaml @@ -0,0 +1,5 @@ +pr: 87309 +summary: "Script: Add Metadata to ingest context" +area: Infra/Scripting +type: enhancement +issues: [] From a9d6267913535c5a2829130fdde2a392dcb6a99f Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 1 Jun 2022 18:39:37 -0500 Subject: [PATCH 03/15] spotless --- .../java/org/elasticsearch/script/IngestScriptMetadataTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java index 436bc41ce3dbe..cdc66f07b6136 100644 --- a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java @@ -24,7 +24,6 @@ public void testVersionType() { assertEquals(VersionType.toString(vt), ctx.get("_version_type")); } - for (VersionType vt : VersionType.values()) { ctx.put("_version_type", VersionType.toString(vt)); assertEquals(vt, m.getVersionType()); From f02f66a3211e5cdd5e18f7eb039fa01fedc30aaa Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 6 Jun 2022 13:16:55 -0500 Subject: [PATCH 04/15] use primitive long for version --- .../test/ingest/190_script_processor.yml | 2 +- .../org.elasticsearch.script.ingest.meta.txt | 6 ++++-- .../org/elasticsearch/script/IngestScript.java | 18 +++++++++++++++--- .../script/field/MapBackedMetadata.java | 6 +++++- .../script/IngestScriptMetadataTests.java | 11 +++++++++++ .../script/field/MapBackedMetadataTests.java | 3 +++ 6 files changed, 39 insertions(+), 7 deletions(-) diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml index 1d00ea9b5e46d..3fabb5a6f639e 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml @@ -327,7 +327,7 @@ teardown: { "script" : { "lang": "painless", - "source" : "Metadata m = meta(); m.index += '2'; m.id += 'extra'; m.routing = 'myRouting'; m.versionType = VersionType.EXTERNAL; m.version = Long.valueOf(5);" + "source" : "Metadata m = meta(); m.index += '2'; m.id += 'extra'; m.routing = 'myRouting'; m.versionType = VersionType.EXTERNAL; m.version = 5" } } ] diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt index f7ca27cd00e7a..082d539de1993 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt @@ -16,8 +16,10 @@ class org.elasticsearch.script.IngestScript$Metadata @alias[class="Metadata"] { void setId(String) String getRouting() void setRouting(String) - Long getVersion() - void setVersion(Long) + long getVersion() + void setVersion(long) + void removeVersion() + boolean hasVersion() VersionType getVersionType() void setVersionType(VersionType) ZonedDateTime getTimestamp() diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java index 089a3859b8021..2baa8c46dc032 100644 --- a/server/src/main/java/org/elasticsearch/script/IngestScript.java +++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java @@ -100,11 +100,23 @@ public void setRouting(String routing) { store.setRouting(routing); } - public Long getVersion() { - return store.getVersion(); + public boolean hasVersion() { + return store.getRawVersion() != null; } - public void setVersion(Long version) { + public void removeVersion() { + store.removeVersion(); + } + + public long getVersion() { + Long version = store.getVersion(); + if (version == null) { + return Long.MIN_VALUE; + } + return version; + } + + public void setVersion(long version) { store.setVersion(version); } diff --git a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java index a1dd670836262..552587e44ad6f 100644 --- a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java +++ b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java @@ -66,7 +66,7 @@ public Long getVersion() { ); } - protected Object getRawVersion() { + public Object getRawVersion() { return map.get(VERSION); } @@ -74,6 +74,10 @@ public MapBackedMetadata setVersion(Long version) { return set(VERSION, version); } + public void removeVersion() { + map.remove(VERSION); + } + public String getRouting() { return getString(ROUTING); } diff --git a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java index cdc66f07b6136..093efa504c85e 100644 --- a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java @@ -32,4 +32,15 @@ public void testVersionType() { m.setVersionType(null); assertNull(m.getVersionType()); } + + public void testVersion() { + Map ctx = new HashMap<>(); + IngestScript.Metadata m = new IngestScript.Metadata(ctx, null); + assertEquals(Long.MIN_VALUE, m.getVersion()); + assertFalse(m.hasVersion()); + assertFalse(m.hasVersion()); + ctx.put("_version", 5); + assertTrue(m.hasVersion()); + assertEquals(5, m.getVersion()); + } } diff --git a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java index c1f4adc36b453..b4abf4a2a89d0 100644 --- a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java @@ -85,6 +85,9 @@ public void testVersion() { assertEquals(Long.valueOf(version), m.getVersion()); assertEquals(version, m.getRawVersion()); assertEquals(version, m.getMap().get("_version")); + m.removeVersion(); + assertNull(m.getMap().get("_version")); + assertNull(m.getVersion()); String badVersion = "badVersion"; m.set("_version", badVersion); From e352d3d23ebb9898ec3fd97578ce748a0fd638ea Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 9 Jun 2022 09:18:36 -0500 Subject: [PATCH 05/15] meta() -> metadata(), add some javadocs --- .../rest-api-spec/test/ingest/190_script_processor.yml | 2 +- .../painless/org.elasticsearch.script.ingest.meta.txt | 2 +- .../main/java/org/elasticsearch/script/IngestScript.java | 4 +++- .../elasticsearch/script/field/MapBackedMetadata.java | 9 +++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml index 5ea6ffbf1b527..cf190175dbdca 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml @@ -327,7 +327,7 @@ teardown: { "script" : { "lang": "painless", - "source" : "Metadata m = meta(); m.index += '2'; m.id += 'extra'; m.routing = 'myRouting'; m.versionType = VersionType.EXTERNAL; m.version = 5" + "source" : "Metadata m = metadata(); m.index += '2'; m.id += 'extra'; m.routing = 'myRouting'; m.versionType = VersionType.EXTERNAL; m.version = 5" } } ] diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt index 082d539de1993..802962ba08271 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt @@ -26,7 +26,7 @@ class org.elasticsearch.script.IngestScript$Metadata @alias[class="Metadata"] { } class org.elasticsearch.script.IngestScript { - Metadata meta() + Metadata metadata() } class org.elasticsearch.index.VersionType { diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java index 2baa8c46dc032..ca27d177d452d 100644 --- a/server/src/main/java/org/elasticsearch/script/IngestScript.java +++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java @@ -49,11 +49,13 @@ public Map getParams() { return params; } + /** Provides backwards compatibility access to ctx */ public Map getCtx() { return metadata != null ? metadata.store.getMap() : null; } - public Metadata meta() { + /** Return the ingest metadata object */ + public Metadata metadata() { return metadata; } diff --git a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java index 552587e44ad6f..3c0a0aba662a7 100644 --- a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java +++ b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java @@ -21,6 +21,15 @@ public class MapBackedMetadata { public static final String ROUTING = "_routing"; public static final String SOURCE = "_source"; + /** + * The backing map has the metadata keys from above. For backwards compatibility this map is exposed to + * scripts and they can modify any number of keys. + * + * Metadata keys are protected via getters and setters in this class but may be modified externally. + * + * Users of the class may add additional metadata keys and helpers by using the {@link #set(String, Object)}, + * {@link #getString(String)} or accessing the underlying map directly. + */ protected Map map; public MapBackedMetadata(Map map) { From 41273d0787e10ab72dceae39a021f8b82a826430 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 9 Jun 2022 10:15:52 -0500 Subject: [PATCH 06/15] Remove chaining, remove map size constructor --- .../script/field/MapBackedMetadata.java | 23 ++++++++----------- .../script/field/MapBackedMetadataTests.java | 17 +++++++++----- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java index 3c0a0aba662a7..8f104b9beaab7 100644 --- a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java +++ b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java @@ -36,24 +36,20 @@ public MapBackedMetadata(Map map) { this.map = map; } - public MapBackedMetadata(int size) { - this.map = Maps.newMapWithExpectedSize(size); - } - public String getIndex() { return getString(INDEX); } - public MapBackedMetadata setIndex(String index) { - return set(INDEX, index); + public void setIndex(String index) { + set(INDEX, index); } public String getId() { return getString(ID); } - public MapBackedMetadata setId(String id) { - return set(ID, id); + public void setId(String id) { + set(ID, id); } public Long getVersion() { @@ -79,8 +75,8 @@ public Object getRawVersion() { return map.get(VERSION); } - public MapBackedMetadata setVersion(Long version) { - return set(VERSION, version); + public void setVersion(Long version) { +set(VERSION, version); } public void removeVersion() { @@ -91,17 +87,16 @@ public String getRouting() { return getString(ROUTING); } - public MapBackedMetadata setRouting(String routing) { - return set(ROUTING, routing); + public void setRouting(String routing) { + set(ROUTING, routing); } public String getString(String key) { return Objects.toString(map.get(key), null); } - public MapBackedMetadata set(String key, Object value) { + public void set(String key, Object value) { map.put(key, value); - return this; } public Map getMap() { diff --git a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java index b4abf4a2a89d0..ad70b40479e97 100644 --- a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.script.field; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -15,7 +16,7 @@ public class MapBackedMetadataTests extends ESTestCase { public void testString() { - MapBackedMetadata m = new MapBackedMetadata(1); + MapBackedMetadata m = singleElementMetadata(); String key = "myKey"; String value = "myValue"; assertNull(m.getString(key)); @@ -44,7 +45,7 @@ public void testString() { public void testIndex() { String index = "myIndex"; - MapBackedMetadata m = new MapBackedMetadata(1); + MapBackedMetadata m = singleElementMetadata(); assertNull(m.getIndex()); assertNull(m.getMap().get("_index")); @@ -55,7 +56,7 @@ public void testIndex() { public void testId() { String id = "myId"; - MapBackedMetadata m = new MapBackedMetadata(1); + MapBackedMetadata m = singleElementMetadata(); assertNull(m.getId()); assertNull(m.getMap().get("_id")); @@ -66,7 +67,7 @@ public void testId() { public void testRouting() { String routing = "myRouting"; - MapBackedMetadata m = new MapBackedMetadata(1); + MapBackedMetadata m = singleElementMetadata(); assertNull(m.getRouting()); assertNull(m.getMap().get("_routing")); @@ -77,7 +78,7 @@ public void testRouting() { public void testVersion() { long version = 500; - MapBackedMetadata m = new MapBackedMetadata(1); + MapBackedMetadata m = singleElementMetadata(); assertNull(m.getVersion()); assertNull(m.getMap().get("_version")); @@ -108,7 +109,7 @@ public void testVersion() { } public void testSource() { - MapBackedMetadata m = new MapBackedMetadata(1); + MapBackedMetadata m = singleElementMetadata(); Map source = new HashMap<>(); source.put("foo", "bar"); @@ -142,4 +143,8 @@ public void testCtx() { assertEquals("myRouting", m.getRouting()); assertEquals("bar", m.getSource().get("foo")); } + + private MapBackedMetadata singleElementMetadata() { + return new MapBackedMetadata(Maps.newMapWithExpectedSize(1)); + } } From ae37ff60f885ab2e21f0efd9a9dd9f6b0cbcf0e5 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 9 Jun 2022 10:37:49 -0500 Subject: [PATCH 07/15] rm unused import --- .../java/org/elasticsearch/script/field/MapBackedMetadata.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java index 8f104b9beaab7..efed5f61aa333 100644 --- a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java +++ b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java @@ -8,8 +8,6 @@ package org.elasticsearch.script.field; -import org.elasticsearch.common.util.Maps; - import java.util.Map; import java.util.Objects; From ceae2d394b66bb094523e68be727fd4bf2e177f5 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 9 Jun 2022 10:58:09 -0500 Subject: [PATCH 08/15] spotless --- .../java/org/elasticsearch/script/field/MapBackedMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java index efed5f61aa333..ae43c140e64bd 100644 --- a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java +++ b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java @@ -74,7 +74,7 @@ public Object getRawVersion() { } public void setVersion(Long version) { -set(VERSION, version); + set(VERSION, version); } public void removeVersion() { From 43e4b56bc21c179b46ca68e083de3c3814dc4e98 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 13 Jun 2022 10:11:11 -0500 Subject: [PATCH 09/15] remove meta.txt, change Metadata.versionType to a string --- .../org.elasticsearch.script.ingest.meta.txt | 36 ------------------- .../org.elasticsearch.script.ingest.txt | 25 +++++++++++++ .../elasticsearch/script/IngestScript.java | 14 +++++--- .../script/IngestScriptMetadataTests.java | 7 ++-- 4 files changed, 38 insertions(+), 44 deletions(-) delete mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt deleted file mode 100644 index 802962ba08271..0000000000000 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.meta.txt +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -# This file only whitelists the script class and the Metadata class for ingest scripts. -# Due to name conflicts, this whitelist cannot be loaded with other scripts with conflicting Metadata types. - -class org.elasticsearch.script.IngestScript$Metadata @alias[class="Metadata"] { - String getIndex() - void setIndex(String) - String getId() - void setId(String) - String getRouting() - void setRouting(String) - long getVersion() - void setVersion(long) - void removeVersion() - boolean hasVersion() - VersionType getVersionType() - void setVersionType(VersionType) - ZonedDateTime getTimestamp() -} - -class org.elasticsearch.script.IngestScript { - Metadata metadata() -} - -class org.elasticsearch.index.VersionType { - VersionType INTERNAL - VersionType EXTERNAL - VersionType EXTERNAL_GTE -} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt index 72288e84ff89b..71d6eaf8cafc3 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt @@ -18,3 +18,28 @@ class org.elasticsearch.painless.api.Json { String dump(def) String dump(def, boolean) } + +class org.elasticsearch.script.IngestScript$Metadata @alias[class="Metadata"] { + String getIndex() + void setIndex(String) + + String getId() + void setId(String) + + String getRouting() + void setRouting(String) + + long getVersion() + void setVersion(long) + void removeVersion() + boolean hasVersion() + + String getVersionType() + void setVersionType(String) + + ZonedDateTime getTimestamp() +} + +class org.elasticsearch.script.IngestScript { + Metadata metadata() +} diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java index ca27d177d452d..06dd3c20427a3 100644 --- a/server/src/main/java/org/elasticsearch/script/IngestScript.java +++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java @@ -122,13 +122,17 @@ public void setVersion(long version) { store.setVersion(version); } - public VersionType getVersionType() { - String str = store.getString(VERSION_TYPE); - return str != null ? VersionType.fromString(str.toLowerCase(Locale.ROOT)) : null; + public String getVersionType() { + return store.getString(VERSION_TYPE); } - public void setVersionType(VersionType versionType) { - store.set(VERSION_TYPE, versionType != null ? VersionType.toString(versionType) : null); + public void setVersionType(String versionType) { + if (versionType != null) { + versionType = versionType.toLowerCase(Locale.ROOT); + // validation + VersionType.fromString(versionType); + } + store.set(VERSION_TYPE, versionType); } public ZonedDateTime getTimestamp() { diff --git a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java index 093efa504c85e..a20c26f157ebe 100644 --- a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.ESTestCase; import java.util.HashMap; +import java.util.Locale; import java.util.Map; public class IngestScriptMetadataTests extends ESTestCase { @@ -20,13 +21,13 @@ public void testVersionType() { IngestScript.Metadata m = new IngestScript.Metadata(ctx, null); assertNull(m.getVersionType()); for (VersionType vt : VersionType.values()) { - m.setVersionType(vt); - assertEquals(VersionType.toString(vt), ctx.get("_version_type")); + m.setVersionType(vt.toString()); + assertEquals(VersionType.toString(vt).toLowerCase(Locale.ROOT), ctx.get("_version_type")); } for (VersionType vt : VersionType.values()) { ctx.put("_version_type", VersionType.toString(vt)); - assertEquals(vt, m.getVersionType()); + assertEquals(vt.toString().toLowerCase(Locale.ROOT), m.getVersionType()); } m.setVersionType(null); From 64539fcee0e6c17d325470bd47b8756d9f0bc50c Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 13 Jun 2022 10:24:04 -0500 Subject: [PATCH 10/15] Update yamlRestTest with string versionType --- .../rest-api-spec/test/ingest/190_script_processor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml index cf190175dbdca..540ce9203b9c2 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml @@ -327,7 +327,7 @@ teardown: { "script" : { "lang": "painless", - "source" : "Metadata m = metadata(); m.index += '2'; m.id += 'extra'; m.routing = 'myRouting'; m.versionType = VersionType.EXTERNAL; m.version = 5" + "source" : "Metadata m = metadata(); m.index += '2'; m.id += 'extra'; m.routing = 'myRouting'; m.versionType = 'external'; m.version = 5" } } ] From ab5155130da0bd6681bf35ee7c160bec016b3043 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 13 Jun 2022 10:38:47 -0500 Subject: [PATCH 11/15] rm removeVersion and hasVersion --- .../painless/org.elasticsearch.script.ingest.txt | 2 -- .../java/org/elasticsearch/script/IngestScript.java | 8 -------- .../elasticsearch/script/field/MapBackedMetadata.java | 4 ---- .../script/IngestScriptMetadataTests.java | 11 ----------- .../script/field/MapBackedMetadataTests.java | 3 --- 5 files changed, 28 deletions(-) diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt index 71d6eaf8cafc3..a8cb2171d200b 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt @@ -31,8 +31,6 @@ class org.elasticsearch.script.IngestScript$Metadata @alias[class="Metadata"] { long getVersion() void setVersion(long) - void removeVersion() - boolean hasVersion() String getVersionType() void setVersionType(String) diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java index 06dd3c20427a3..330b9472a4eb1 100644 --- a/server/src/main/java/org/elasticsearch/script/IngestScript.java +++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java @@ -102,14 +102,6 @@ public void setRouting(String routing) { store.setRouting(routing); } - public boolean hasVersion() { - return store.getRawVersion() != null; - } - - public void removeVersion() { - store.removeVersion(); - } - public long getVersion() { Long version = store.getVersion(); if (version == null) { diff --git a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java index ae43c140e64bd..24f00b515213d 100644 --- a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java +++ b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java @@ -77,10 +77,6 @@ public void setVersion(Long version) { set(VERSION, version); } - public void removeVersion() { - map.remove(VERSION); - } - public String getRouting() { return getString(ROUTING); } diff --git a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java index a20c26f157ebe..05b0aebe8ffed 100644 --- a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java @@ -33,15 +33,4 @@ public void testVersionType() { m.setVersionType(null); assertNull(m.getVersionType()); } - - public void testVersion() { - Map ctx = new HashMap<>(); - IngestScript.Metadata m = new IngestScript.Metadata(ctx, null); - assertEquals(Long.MIN_VALUE, m.getVersion()); - assertFalse(m.hasVersion()); - assertFalse(m.hasVersion()); - ctx.put("_version", 5); - assertTrue(m.hasVersion()); - assertEquals(5, m.getVersion()); - } } diff --git a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java index ad70b40479e97..7bd41db50bad5 100644 --- a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java @@ -86,9 +86,6 @@ public void testVersion() { assertEquals(Long.valueOf(version), m.getVersion()); assertEquals(version, m.getRawVersion()); assertEquals(version, m.getMap().get("_version")); - m.removeVersion(); - assertNull(m.getMap().get("_version")); - assertNull(m.getVersion()); String badVersion = "badVersion"; m.set("_version", badVersion); From 8c308c59e91d1c1fee64c3fc893444caeaa31d00 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 28 Jun 2022 12:44:22 -0500 Subject: [PATCH 12/15] Move test and use IngestSourceAndMetadata as Metadata --- .../ingest/common/ScriptProcessor.java | 3 +- .../common/ScriptProcessorFactoryTests.java | 6 +- .../org.elasticsearch.script.ingest.txt | 2 +- .../ingest/WriteableIngestDocument.java | 5 +- .../elasticsearch/ingest/IngestDocument.java | 25 +-- .../ingest/IngestSourceAndMetadata.java | 3 +- .../elasticsearch/script/IngestScript.java | 73 +-------- .../org/elasticsearch/script/Metadata.java | 39 +++++ .../script/field/MapBackedMetadata.java | 114 -------------- .../ingest/IngestSourceAndMetadataTests.java | 20 +++ .../script/IngestScriptMetadataTests.java | 36 ----- .../script/field/MapBackedMetadataTests.java | 147 ------------------ 12 files changed, 72 insertions(+), 401 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/script/Metadata.java delete mode 100644 server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java delete mode 100644 server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java delete mode 100644 server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index 8037a97b55f7f..b400929ef9179 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -75,8 +75,7 @@ public IngestDocument execute(IngestDocument document) { if (factory == null) { factory = scriptService.compile(script, IngestScript.CONTEXT); } - factory.newInstance(script.getParams(), new IngestScript.Metadata(document.getSourceAndMetadata(), document.getTimestamp())) - .execute(); + factory.newInstance(script.getParams(), document.getIngestSourceAndMetadata()).execute(); return document; } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index 897a7ab9afae2..f638e5a9e4841 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -10,7 +10,9 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.ingest.TestIngestDocument; import org.elasticsearch.script.IngestScript; +import org.elasticsearch.script.Metadata; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptException; @@ -156,8 +158,8 @@ public void testInlineIsCompiled() throws Exception { assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE)); assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap())); assertNotNull(processor.getPrecompiledIngestScriptFactory()); - Map ctx = new HashMap<>(); - processor.getPrecompiledIngestScriptFactory().newInstance(null, new IngestScript.Metadata(ctx, null)).execute(); + Metadata ctx = TestIngestDocument.emptyIngestDocument().getIngestSourceAndMetadata(); + processor.getPrecompiledIngestScriptFactory().newInstance(null, ctx).execute(); assertThat(ctx.get("foo"), equalTo("bar")); } diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt index a8cb2171d200b..4924f93e87c91 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.ingest.txt @@ -19,7 +19,7 @@ class org.elasticsearch.painless.api.Json { String dump(def, boolean) } -class org.elasticsearch.script.IngestScript$Metadata @alias[class="Metadata"] { +class org.elasticsearch.script.Metadata { String getIndex() void setIndex(String) diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java index e22f4cb1b6f4a..c83f6ed1c95b7 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java @@ -42,7 +42,9 @@ final class WriteableIngestDocument implements Writeable, ToXContentFragment { "ingest_document", true, a -> { - Map sourceAndMetadata = Maps.newHashMapWithExpectedSize(5); + Map source = (Map) a[5]; + Map sourceAndMetadata = Maps.newHashMapWithExpectedSize(5 + source.size()); + sourceAndMetadata.putAll(source); sourceAndMetadata.put(Metadata.INDEX.getFieldName(), a[0]); sourceAndMetadata.put(Metadata.ID.getFieldName(), a[1]); if (a[2] != null) { @@ -54,7 +56,6 @@ final class WriteableIngestDocument implements Writeable, ToXContentFragment { if (a[4] != null) { sourceAndMetadata.put(Metadata.VERSION_TYPE.getFieldName(), a[4]); } - sourceAndMetadata.putAll((Map) a[5]); Map ingestMetadata = (Map) a[6]; return new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, ingestMetadata)); } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index accd97b38eae3..f1bc994427f02 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -727,7 +727,7 @@ public Map getSourceAndMetadata() { /** * Get source and metadata map as {@link IngestSourceAndMetadata} */ - IngestSourceAndMetadata getIngestSourceAndMetadata() { + public IngestSourceAndMetadata getIngestSourceAndMetadata() { return sourceAndMetadata; } @@ -753,29 +753,6 @@ public Map getIngestMetadata() { return this.ingestMetadata; } - /** - * Returns the document including its metadata fields, unless {@link #extractMetadata()} has been called, in which case the - * metadata fields will not be present anymore. - * Modify the document instead using {@link #setFieldValue(String, Object)} and {@link #removeField(String)} - */ - public Map getSourceAndMetadata() { - return this.sourceAndMetadata; - } - - /** - * Fetch the timestamp from the ingestMetadata, if it exists - * @return the timestamp for the document or null - */ - public ZonedDateTime getTimestamp() { - if (ingestMetadata == null) { - return null; - } - if (ingestMetadata.get(TIMESTAMP)instanceof ZonedDateTime timestamp) { - return timestamp; - } - return null; - } - @SuppressWarnings("unchecked") public static Map deepCopyMap(Map source) { return (Map) deepCopy(source); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java index 1312e4bc45d69..c0c513e5fe1fd 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; +import org.elasticsearch.script.Metadata; import java.time.ZonedDateTime; import java.util.AbstractCollection; @@ -40,7 +41,7 @@ * * The map is expected to be used by processors, server code should the typed getter and setters where possible. */ -class IngestSourceAndMetadata extends AbstractMap { +class IngestSourceAndMetadata extends AbstractMap implements Metadata { protected final ZonedDateTime timestamp; /** diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java index 330b9472a4eb1..ef13830e097f7 100644 --- a/server/src/main/java/org/elasticsearch/script/IngestScript.java +++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java @@ -10,11 +10,7 @@ package org.elasticsearch.script; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.script.field.MapBackedMetadata; -import java.time.ZonedDateTime; -import java.util.Locale; import java.util.Map; /** @@ -51,7 +47,7 @@ public Map getParams() { /** Provides backwards compatibility access to ctx */ public Map getCtx() { - return metadata != null ? metadata.store.getMap() : null; + return metadata; } /** Return the ingest metadata object */ @@ -64,71 +60,4 @@ public Metadata metadata() { public interface Factory { IngestScript newInstance(Map params, Metadata metadata); } - - /** - * Metadata available to scripts, backed by the ctx map. - */ - public static class Metadata { - private final MapBackedMetadata store; - private final ZonedDateTime timestamp; - public static final String VERSION_TYPE = "_version_type"; - - public Metadata(Map ctx, ZonedDateTime timestamp) { - store = new MapBackedMetadata(ctx); - this.timestamp = timestamp; - } - - public String getIndex() { - return store.getIndex(); - } - - public void setIndex(String index) { - store.setIndex(index); - } - - public String getId() { - return store.getId(); - } - - public void setId(String id) { - store.setId(id); - } - - public String getRouting() { - return store.getRouting(); - } - - public void setRouting(String routing) { - store.setRouting(routing); - } - - public long getVersion() { - Long version = store.getVersion(); - if (version == null) { - return Long.MIN_VALUE; - } - return version; - } - - public void setVersion(long version) { - store.setVersion(version); - } - - public String getVersionType() { - return store.getString(VERSION_TYPE); - } - - public void setVersionType(String versionType) { - if (versionType != null) { - versionType = versionType.toLowerCase(Locale.ROOT); - // validation - VersionType.fromString(versionType); - } - store.set(VERSION_TYPE, versionType); - } - - public ZonedDateTime getTimestamp() { - return timestamp; - } - } } diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java new file mode 100644 index 0000000000000..ac3e3a16eb5fe --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import java.time.ZonedDateTime; +import java.util.Map; + +/** + * Metadata availabe to write scripts + */ +public interface Metadata extends Map { + String getIndex(); + + void setIndex(String index); + + String getId(); + + void setId(String id); + + String getRouting(); + + void setRouting(String routing); + + long getVersion(); + + void setVersion(long version); + + String getVersionType(); + + void setVersionType(String versionType); + + ZonedDateTime getTimestamp(); +} diff --git a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java b/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java deleted file mode 100644 index 24f00b515213d..0000000000000 --- a/server/src/main/java/org/elasticsearch/script/field/MapBackedMetadata.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script.field; - -import java.util.Map; -import java.util.Objects; - -/** Metadata storage backed by a Map for compatibility with the ctx Map used by ingest and update scripts. */ -public class MapBackedMetadata { - public static final String INDEX = "_index"; - public static final String ID = "_id"; - public static final String VERSION = "_version"; - public static final String ROUTING = "_routing"; - public static final String SOURCE = "_source"; - - /** - * The backing map has the metadata keys from above. For backwards compatibility this map is exposed to - * scripts and they can modify any number of keys. - * - * Metadata keys are protected via getters and setters in this class but may be modified externally. - * - * Users of the class may add additional metadata keys and helpers by using the {@link #set(String, Object)}, - * {@link #getString(String)} or accessing the underlying map directly. - */ - protected Map map; - - public MapBackedMetadata(Map map) { - this.map = map; - } - - public String getIndex() { - return getString(INDEX); - } - - public void setIndex(String index) { - set(INDEX, index); - } - - public String getId() { - return getString(ID); - } - - public void setId(String id) { - set(ID, id); - } - - public Long getVersion() { - Object obj = getRawVersion(); - if (obj == null) { - return null; - } else if (obj instanceof Number number) { - long version = number.longValue(); - if (number.doubleValue() != version) { - // did we round? - throw new IllegalArgumentException( - "version may only be set to an int or a long but was [" + number + "] with type [" + obj.getClass().getName() + "]" - ); - } - return version; - } - throw new IllegalArgumentException( - "version may only be set to an int or a long but was [" + obj + "] with type [" + obj.getClass().getName() + "]" - ); - } - - public Object getRawVersion() { - return map.get(VERSION); - } - - public void setVersion(Long version) { - set(VERSION, version); - } - - public String getRouting() { - return getString(ROUTING); - } - - public void setRouting(String routing) { - set(ROUTING, routing); - } - - public String getString(String key) { - return Objects.toString(map.get(key), null); - } - - public void set(String key, Object value) { - map.put(key, value); - } - - public Map getMap() { - return map; - } - - public MapBackedMetadata setSource(Map source) { - map.put(SOURCE, source); - return this; - } - - @SuppressWarnings("unchecked") - public Map getSource() { - Object source = map.get(SOURCE); - if (source instanceof Map map) { - return (Map) map; - } else { - throw new IllegalArgumentException("source should be a map, not [" + source + "] with [" + source.getClass().getName() + "]"); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java index 5d2c45b6e80c3..6a50ee38f1919 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java @@ -13,6 +13,7 @@ import java.util.HashMap; import java.util.Iterator; +import java.util.Locale; import java.util.Map; import java.util.Set; @@ -319,6 +320,25 @@ public void testDefaultValidatorForAllMetadata() { assertEquals(IngestDocument.Metadata.values().length, IngestSourceAndMetadata.VALIDATORS.size()); } + public void testHandlesAllVersionTypes() { + Map md = new HashMap<>(); + md.put("_version", 1234); + map = new IngestSourceAndMetadata(new HashMap<>(), md, null, null); + assertNull(map.getVersionType()); + for (VersionType vt : VersionType.values()) { + map.setVersionType(VersionType.toString(vt)); + assertEquals(VersionType.toString(vt), map.get("_version_type")); + } + + for (VersionType vt : VersionType.values()) { + map.put("_version_type", VersionType.toString(vt)); + assertEquals(vt.toString().toLowerCase(Locale.ROOT), map.getVersionType()); + } + + map.setVersionType(null); + assertNull(map.getVersionType()); + } + private static class TestEntry implements Map.Entry { String key; Object value; diff --git a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java b/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java deleted file mode 100644 index 05b0aebe8ffed..0000000000000 --- a/server/src/test/java/org/elasticsearch/script/IngestScriptMetadataTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.index.VersionType; -import org.elasticsearch.test.ESTestCase; - -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; - -public class IngestScriptMetadataTests extends ESTestCase { - public void testVersionType() { - Map ctx = new HashMap<>(); - IngestScript.Metadata m = new IngestScript.Metadata(ctx, null); - assertNull(m.getVersionType()); - for (VersionType vt : VersionType.values()) { - m.setVersionType(vt.toString()); - assertEquals(VersionType.toString(vt).toLowerCase(Locale.ROOT), ctx.get("_version_type")); - } - - for (VersionType vt : VersionType.values()) { - ctx.put("_version_type", VersionType.toString(vt)); - assertEquals(vt.toString().toLowerCase(Locale.ROOT), m.getVersionType()); - } - - m.setVersionType(null); - assertNull(m.getVersionType()); - } -} diff --git a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java b/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java deleted file mode 100644 index 7bd41db50bad5..0000000000000 --- a/server/src/test/java/org/elasticsearch/script/field/MapBackedMetadataTests.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script.field; - -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.test.ESTestCase; - -import java.util.HashMap; -import java.util.Map; - -public class MapBackedMetadataTests extends ESTestCase { - public void testString() { - MapBackedMetadata m = singleElementMetadata(); - String key = "myKey"; - String value = "myValue"; - assertNull(m.getString(key)); - assertNull(m.getMap().get(key)); - - m.set(key, value); - assertEquals(value, m.getString(key)); - assertEquals(value, m.getMap().get(key)); - - m.set(key, 1); - assertEquals("1", m.getString(key)); - assertEquals(1, m.getMap().get(key)); - - m.set(key, null); - assertNull(m.getString(key)); - assertNull(m.getMap().get(key)); - - m.set(key, value); - assertEquals(value, m.getString(key)); - assertEquals(value, m.getMap().get(key)); - - m.getMap().remove(key); - assertNull(m.getString(key)); - assertNull(m.getMap().get(key)); - } - - public void testIndex() { - String index = "myIndex"; - MapBackedMetadata m = singleElementMetadata(); - assertNull(m.getIndex()); - assertNull(m.getMap().get("_index")); - - m.setIndex(index); - assertEquals(index, m.getIndex()); - assertEquals(index, m.getMap().get("_index")); - } - - public void testId() { - String id = "myId"; - MapBackedMetadata m = singleElementMetadata(); - assertNull(m.getId()); - assertNull(m.getMap().get("_id")); - - m.setId(id); - assertEquals(id, m.getId()); - assertEquals(id, m.getMap().get("_id")); - } - - public void testRouting() { - String routing = "myRouting"; - MapBackedMetadata m = singleElementMetadata(); - assertNull(m.getRouting()); - assertNull(m.getMap().get("_routing")); - - m.setRouting(routing); - assertEquals(routing, m.getRouting()); - assertEquals(routing, m.getMap().get("_routing")); - } - - public void testVersion() { - long version = 500; - MapBackedMetadata m = singleElementMetadata(); - assertNull(m.getVersion()); - assertNull(m.getMap().get("_version")); - - m.setVersion(version); - assertEquals(Long.valueOf(version), m.getVersion()); - assertEquals(version, m.getRawVersion()); - assertEquals(version, m.getMap().get("_version")); - - String badVersion = "badVersion"; - m.set("_version", badVersion); - assertEquals(badVersion, m.getRawVersion()); - IllegalArgumentException err = expectThrows(IllegalArgumentException.class, m::getVersion); - assertEquals("version may only be set to an int or a long but was [badVersion] with type [java.lang.String]", err.getMessage()); - - double tooBig = Double.MAX_VALUE; - m.set("_version", tooBig); - assertEquals(tooBig, m.getRawVersion()); - err = expectThrows(IllegalArgumentException.class, m::getVersion); - assertEquals("version may only be set to an int or a long but was [" + tooBig + "] with type [java.lang.Double]", err.getMessage()); - - long justRight = (long) 1 << 52; - m.set("_version", (double) justRight); - assertEquals(Long.valueOf(justRight), m.getVersion()); - assertEquals((double) justRight, m.getRawVersion()); - } - - public void testSource() { - MapBackedMetadata m = singleElementMetadata(); - Map source = new HashMap<>(); - source.put("foo", "bar"); - - m.setSource(source); - assertEquals(source, m.getSource()); - assertEquals(source, m.getMap().get("_source")); - - source.put("baz", "qux"); - assertEquals("qux", m.getSource().get("baz")); - - m.getMap().put("_source", "mySource"); - - IllegalArgumentException err = expectThrows(IllegalArgumentException.class, m::getSource); - assertEquals("source should be a map, not [mySource] with [java.lang.String]", err.getMessage()); - } - - public void testCtx() { - Map ctx = new HashMap<>(); - ctx.put("_index", "myIndex"); - ctx.put("_id", "myId"); - ctx.put("_version", 200); - ctx.put("_routing", "myRouting"); - Map source = new HashMap<>(); - source.put("foo", "bar"); - ctx.put("_source", source); - - MapBackedMetadata m = new MapBackedMetadata(ctx); - assertEquals("myIndex", m.getIndex()); - assertEquals("myId", m.getId()); - assertEquals(Long.valueOf(200), m.getVersion()); - assertEquals("myRouting", m.getRouting()); - assertEquals("bar", m.getSource().get("foo")); - } - - private MapBackedMetadata singleElementMetadata() { - return new MapBackedMetadata(Maps.newMapWithExpectedSize(1)); - } -} From 45cfa2b14cf8ad17c22ff150f267dd88d159322b Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 28 Jun 2022 12:50:25 -0500 Subject: [PATCH 13/15] Better javadoc on Metadata.java --- .../org/elasticsearch/script/Metadata.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index ac3e3a16eb5fe..e1795f34e9a26 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -12,28 +12,50 @@ import java.util.Map; /** - * Metadata availabe to write scripts + * Ingest and update metadata available to write scripts */ public interface Metadata extends Map { + /** + * The destination index + */ String getIndex(); void setIndex(String index); + /** + * The document id + */ String getId(); void setId(String id); + /** + * The document routing string + */ String getRouting(); void setRouting(String routing); + /** + * The version of the document + */ long getVersion(); void setVersion(long version); + /** + * The version type of the document, {@link org.elasticsearch.index.VersionType} as a lower-case string. + */ String getVersionType(); + /** + * Set the version type of the document. + * @param versionType {@link org.elasticsearch.index.VersionType} as a lower-case string + */ void setVersionType(String versionType); + /** + * Timestamp of this ingestion or update + */ ZonedDateTime getTimestamp(); } From 942b45d00d6ea2afaa61a5f49c69fe26b12663c1 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 29 Jun 2022 14:51:13 -0500 Subject: [PATCH 14/15] Metadata doesn't extend Map, pass ctx and Metadata separately --- .../elasticsearch/ingest/common/ScriptProcessor.java | 2 +- .../ingest/common/ScriptProcessorFactoryTests.java | 7 ++++--- .../java/org/elasticsearch/ingest/IngestClientIT.java | 2 +- .../action/ingest/WriteableIngestDocument.java | 2 +- .../java/org/elasticsearch/ingest/IngestDocument.java | 9 ++++++++- .../java/org/elasticsearch/script/IngestScript.java | 11 ++++++++--- .../main/java/org/elasticsearch/script/Metadata.java | 3 +-- .../action/ingest/SimulateExecutionServiceTests.java | 2 +- .../ingest/SimulatePipelineRequestParsingTests.java | 6 +++--- .../action/ingest/WriteableIngestDocumentTests.java | 2 +- .../org/elasticsearch/ingest/IngestServiceTests.java | 2 +- .../org/elasticsearch/script/MockScriptEngine.java | 3 +-- .../inference/results/InferenceResultsTestCase.java | 2 +- .../xpack/ml/integration/MlNativeIntegTestCase.java | 2 +- .../elasticsearch/xpack/ml/MlSingleNodeTestCase.java | 2 +- .../xpack/ml/support/BaseMlIntegTestCase.java | 2 +- 16 files changed, 35 insertions(+), 24 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index b400929ef9179..2586b9aed919d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -75,7 +75,7 @@ public IngestDocument execute(IngestDocument document) { if (factory == null) { factory = scriptService.compile(script, IngestScript.CONTEXT); } - factory.newInstance(script.getParams(), document.getIngestSourceAndMetadata()).execute(); + factory.newInstance(script.getParams(), document.getMetadata(), document.getSourceAndMetadata()).execute(); return document; } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index f638e5a9e4841..6fb39fa0fb803 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -10,9 +10,9 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.TestIngestDocument; import org.elasticsearch.script.IngestScript; -import org.elasticsearch.script.Metadata; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptException; @@ -158,8 +158,9 @@ public void testInlineIsCompiled() throws Exception { assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE)); assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap())); assertNotNull(processor.getPrecompiledIngestScriptFactory()); - Metadata ctx = TestIngestDocument.emptyIngestDocument().getIngestSourceAndMetadata(); - processor.getPrecompiledIngestScriptFactory().newInstance(null, ctx).execute(); + IngestDocument doc = TestIngestDocument.emptyIngestDocument(); + Map ctx = TestIngestDocument.emptyIngestDocument().getIngestSourceAndMetadata(); + processor.getPrecompiledIngestScriptFactory().newInstance(null, doc.getMetadata(), ctx).execute(); assertThat(ctx.get("foo"), equalTo("bar")); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java index 8407d72eb728e..2e69cdff0fe80 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java @@ -115,7 +115,7 @@ public void testSimulate() throws Exception { source.put("processed", true); IngestDocument ingestDocument = new IngestDocument("index", "id", Versions.MATCH_ANY, null, null, source); assertThat(simulateDocumentBaseResult.getIngestDocument().getSource(), equalTo(ingestDocument.getSource())); - assertThat(simulateDocumentBaseResult.getIngestDocument().getMetadata(), equalTo(ingestDocument.getMetadata())); + assertThat(simulateDocumentBaseResult.getIngestDocument().getMetadataMap(), equalTo(ingestDocument.getMetadataMap())); assertThat(simulateDocumentBaseResult.getFailure(), nullValue()); // cleanup diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java index c83f6ed1c95b7..e9e2882763e33 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java @@ -107,7 +107,7 @@ IngestDocument getIngestDocument() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(DOC_FIELD); - Map metadataMap = ingestDocument.getMetadata(); + Map metadataMap = ingestDocument.getMetadataMap(); for (Map.Entry metadata : metadataMap.entrySet()) { if (metadata.getValue() != null) { builder.field(metadata.getKey(), metadata.getValue().toString()); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index f1bc994427f02..eaf01089d9226 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -734,10 +734,17 @@ public IngestSourceAndMetadata getIngestSourceAndMetadata() { /** * Get all Metadata values in a Map */ - public Map getMetadata() { + public Map getMetadataMap() { return sourceAndMetadata.getMetadata(); } + /** + * Get the strongly typed metadata + */ + public org.elasticsearch.script.Metadata getMetadata() { + return sourceAndMetadata; + } + /** * Get all source values in a Map */ diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java index ef13830e097f7..87a026caf5361 100644 --- a/server/src/main/java/org/elasticsearch/script/IngestScript.java +++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java @@ -33,11 +33,16 @@ public abstract class IngestScript { /** The generic runtime parameters for the script. */ private final Map params; + /** The metadata available to the script */ private final Metadata metadata; - public IngestScript(Map params, Metadata metadata) { + /** The metadata and source available to the script */ + private final Map ctx; + + public IngestScript(Map params, Metadata metadata, Map ctx) { this.params = params; this.metadata = metadata; + this.ctx = ctx; } /** Return the parameters for this script. */ @@ -47,7 +52,7 @@ public Map getParams() { /** Provides backwards compatibility access to ctx */ public Map getCtx() { - return metadata; + return ctx; } /** Return the ingest metadata object */ @@ -58,6 +63,6 @@ public Metadata metadata() { public abstract void execute(); public interface Factory { - IngestScript newInstance(Map params, Metadata metadata); + IngestScript newInstance(Map params, Metadata metadata, Map ctx); } } diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index e1795f34e9a26..e598a211c109a 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -9,12 +9,11 @@ package org.elasticsearch.script; import java.time.ZonedDateTime; -import java.util.Map; /** * Ingest and update metadata available to write scripts */ -public interface Metadata extends Map { +public interface Metadata { /** * The destination index */ diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java index f609114832a70..958c59f67a838 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java @@ -378,7 +378,7 @@ public boolean isAsync() { for (int id = 0; id < numDocs; id++) { SimulateDocumentBaseResult result = (SimulateDocumentBaseResult) response.getResults().get(id); assertThat( - result.getIngestDocument().getMetadata().get(IngestDocument.Metadata.ID.getFieldName()), + result.getIngestDocument().getMetadataMap().get(IngestDocument.Metadata.ID.getFieldName()), equalTo(Integer.toString(id)) ); assertThat(result.getIngestDocument().getSourceAndMetadata().get("processed"), is(true)); diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 3fbb867346fbc..e906a2d619cb0 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -100,7 +100,7 @@ public void testParseUsingPipelineStore() throws Exception { Iterator> expectedDocsIterator = expectedDocs.iterator(); for (IngestDocument ingestDocument : actualRequest.documents()) { Map expectedDocument = expectedDocsIterator.next(); - Map metadataMap = ingestDocument.getMetadata(); + Map metadataMap = ingestDocument.getMetadataMap(); assertThat(metadataMap.get(INDEX.getFieldName()), equalTo(expectedDocument.get(INDEX.getFieldName()))); assertThat(metadataMap.get(ID.getFieldName()), equalTo(expectedDocument.get(ID.getFieldName()))); assertThat(ingestDocument.getSource(), equalTo(expectedDocument.get(Fields.SOURCE))); @@ -196,7 +196,7 @@ public void testParseWithProvidedPipeline() throws Exception { Iterator> expectedDocsIterator = expectedDocs.iterator(); for (IngestDocument ingestDocument : actualRequest.documents()) { Map expectedDocument = expectedDocsIterator.next(); - Map metadataMap = ingestDocument.getMetadata(); + Map metadataMap = ingestDocument.getMetadataMap(); assertThat(metadataMap.get(INDEX.getFieldName()), equalTo(expectedDocument.get(INDEX.getFieldName()))); assertThat(metadataMap.get(ID.getFieldName()), equalTo(expectedDocument.get(ID.getFieldName()))); assertThat(metadataMap.get(ROUTING.getFieldName()), equalTo(expectedDocument.get(ROUTING.getFieldName()))); @@ -349,7 +349,7 @@ public void testIngestPipelineWithDocumentsWithType() throws Exception { Iterator> expectedDocsIterator = expectedDocs.iterator(); for (IngestDocument ingestDocument : actualRequest.documents()) { Map expectedDocument = expectedDocsIterator.next(); - Map metadataMap = ingestDocument.getMetadata(); + Map metadataMap = ingestDocument.getMetadataMap(); assertThat(metadataMap.get(INDEX.getFieldName()), equalTo(expectedDocument.get(INDEX.getFieldName()))); assertThat(metadataMap.get(ID.getFieldName()), equalTo(expectedDocument.get(ID.getFieldName()))); assertThat(metadataMap.get(ROUTING.getFieldName()), equalTo(expectedDocument.get(ROUTING.getFieldName()))); diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index f3e655fb58583..8c17ff1879a1b 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -142,7 +142,7 @@ public void testToXContent() throws IOException { Map toXContentSource = (Map) toXContentDoc.get("_source"); Map toXContentIngestMetadata = (Map) toXContentDoc.get("_ingest"); - Map metadataMap = ingestDocument.getMetadata(); + Map metadataMap = ingestDocument.getMetadataMap(); for (Map.Entry metadata : metadataMap.entrySet()) { String fieldName = metadata.getKey(); if (metadata.getValue() == null) { diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index c3559aceafa06..4471efe555157 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -2231,7 +2231,7 @@ private class IngestDocumentMatcher implements ArgumentMatcher { public boolean matches(IngestDocument other) { // ingest metadata and IngestSourceAndMetadata will not be the same (timestamp differs every time) return Objects.equals(ingestDocument.getSource(), other.getSource()) - && Objects.equals(ingestDocument.getMetadata(), other.getMetadata()); + && Objects.equals(ingestDocument.getMetadataMap(), other.getMetadataMap()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java index 2e9c9a58bb934..0b9f7f8972620 100644 --- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java +++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java @@ -148,10 +148,9 @@ public boolean needs_score() { } else if (context.instanceClazz.equals(BytesRefSortScript.class)) { return context.factoryClazz.cast(new MockBytesRefSortScriptFactory(script)); } else if (context.instanceClazz.equals(IngestScript.class)) { - IngestScript.Factory factory = (parameters, metadata) -> new IngestScript(parameters, metadata) { + IngestScript.Factory factory = (parameters, metadata, ctx) -> new IngestScript(parameters, metadata, ctx) { @Override public void execute() { - final Map ctx = getCtx(); script.apply(ctx); } }; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResultsTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResultsTestCase.java index 4377051fabd59..1663c63eeeae3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResultsTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResultsTestCase.java @@ -55,7 +55,7 @@ public void testWriteToDocAndSerialize() throws IOException { InferenceResults.writeResult(inferenceResult, document, parentField, modelId); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { builder.startObject(); - Map metadataMap = document.getMetadata(); + Map metadataMap = document.getMetadataMap(); for (Map.Entry metadata : metadataMap.entrySet()) { if (metadata.getValue() != null) { builder.field(metadata.getKey(), metadata.getValue().toString()); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index ec0d0f3453f02..b3928cae4dc90 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -440,7 +440,7 @@ public T compile(String name, String script, ScriptContext context, Map 0.0))); } if (context.name.equals("ingest")) { - IngestScript.Factory factory = (params, metadata) -> new IngestScript(params, metadata) { + IngestScript.Factory factory = (params, metadata, ctx) -> new IngestScript(params, metadata, ctx) { @Override public void execute() {} }; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java index 5a06203e74a05..8749714d971fe 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MlSingleNodeTestCase.java @@ -200,7 +200,7 @@ public T compile(String name, String script, ScriptContext context, Map 0.0))); } if (context.name.equals("ingest")) { - IngestScript.Factory factory = (vars, metadata) -> new IngestScript(vars, metadata) { + IngestScript.Factory factory = (vars, metadata, ctx) -> new IngestScript(vars, metadata, ctx) { @Override public void execute() {} }; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index 885a4cc9a4f48..d2bb3daa13f07 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -554,7 +554,7 @@ public T compile(String name, String script, ScriptContext context, Map 0.0))); } if (context.name.equals("ingest")) { - IngestScript.Factory factory = (vars, metadata) -> new IngestScript(vars, metadata) { + IngestScript.Factory factory = (vars, metadata, ctx) -> new IngestScript(vars, metadata, ctx) { @Override public void execute() {} }; From 8bf32c3314613377105b80438bcd1668bd215236 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 29 Jun 2022 15:19:55 -0500 Subject: [PATCH 15/15] Use getMetadataMap in DotExpanderProcessorTests --- .../ingest/common/DotExpanderProcessorTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java index fc17506555edb..1714717d0e6d3 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java @@ -48,7 +48,7 @@ public void testEscapeFields() throws Exception { processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getSource().size(), equalTo(1)); - assertThat(document.getMetadata().size(), equalTo(1)); // the default version + assertThat(document.getMetadataMap().size(), equalTo(1)); // the default version assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); assertThat(document.getFieldValue("foo.bar.0", String.class), equalTo("baz2")); assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("baz1")); @@ -60,7 +60,7 @@ public void testEscapeFields() throws Exception { processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getSource().size(), equalTo(1)); - assertThat(document.getMetadata().size(), equalTo(1)); // the default version + assertThat(document.getMetadataMap().size(), equalTo(1)); // the default version assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); assertThat(document.getFieldValue("foo.bar.0", Integer.class), equalTo(1)); assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("2"));