From a316c045da8329f24c8a7c04a509e93c38597071 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 18 Jun 2020 12:49:11 -0400 Subject: [PATCH 1/2] Scripted fields: rework script contexts This reworks the script contexts to support multiple values for each hit. Just like before, these are almost certainly not the final implementation, but they give us something to iterate on. --- .../AbstractScriptFieldsScript.java | 11 ++- .../LongArrayScriptFieldScript.java | 34 -------- .../runtimefields/LongScriptFieldScript.java | 23 ++++- .../xpack/runtimefields/RuntimeFields.java | 1 - .../StringScriptFieldsScript.java | 24 ++++- .../LongArrayScriptFieldScriptTests.java | 87 ------------------- .../LongScriptFieldScriptTests.java | 45 ++++++++-- .../ScriptFieldScriptTestCase.java | 8 +- .../StringScriptFieldsScriptTests.java | 56 +++++++++--- 9 files changed, 133 insertions(+), 156 deletions(-) delete mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java delete mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java index 95c419b6ad558..952192c52dbf7 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java @@ -35,7 +35,7 @@ public AbstractScriptFieldsScript(Map params, SourceLookup sourc /** * Set the document to run the script against. */ - public void setDocument(int docId) { + public final void setDocument(int docId) { source.setSegmentAndDocument(ctx, docId); fieldData.setDocument(docId); } @@ -43,21 +43,24 @@ public void setDocument(int docId) { /** * Expose the {@code params} of the script to the script itself. */ - public Map getParams() { + public final Map getParams() { return params; } /** * Expose the {@code _source} to the script. */ - public Map getSource() { + public final Map getSource() { return source; } /** * Expose field data to the script as {@code doc}. */ - public Map> getDoc() { + public final Map> getDoc() { return fieldData; } + + public abstract void execute(); + } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java deleted file mode 100644 index e9676f580b45f..0000000000000 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptFactory; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.Map; - -public abstract class LongArrayScriptFieldScript extends AbstractScriptFieldsScript { - public static final ScriptContext CONTEXT = new ScriptContext<>("long_array_script_field", Factory.class); - public static final String[] PARAMETERS = {}; - - public interface Factory extends ScriptFactory { - LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); - } - public static interface LeafFactory { - LongArrayScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; - } - - public LongArrayScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { - super(params, source, fieldData, ctx); - } - - public abstract long[] execute(); -} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index 4e1dce5c9facc..43d78f724ef5f 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.util.Map; +import java.util.function.LongConsumer; public abstract class LongScriptFieldScript extends AbstractScriptFieldsScript { public static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); @@ -23,12 +24,28 @@ public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } public static interface LeafFactory { - LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException; } - public LongScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + private final LongConsumer sync; + + public LongScriptFieldScript( + Map params, + SourceLookup source, + DocLookup fieldData, + LeafReaderContext ctx, + LongConsumer sync + ) { super(params, source, fieldData, ctx); + this.sync = sync; } - public abstract long execute(); + /** + * Expose the consumer to the script. + *

+ * This is temporary and I'll remove it in the next PR when I figure out class methods. + */ + public LongConsumer getSync() { + return sync; + } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 0a91bd6db2021..777495c834b46 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -26,7 +26,6 @@ public Map getMappers() { @Override public List> getContexts() { return List.of( - LongArrayScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT, StringScriptFieldsScript.CONTEXT ); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java index b8f3809de3af3..0e57878196f46 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.util.Map; +import java.util.function.Consumer; public abstract class StringScriptFieldsScript extends AbstractScriptFieldsScript { public static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); @@ -22,13 +23,30 @@ public abstract class StringScriptFieldsScript extends AbstractScriptFieldsScrip public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } + public static interface LeafFactory { - StringScriptFieldsScript newInstance(LeafReaderContext ctx) throws IOException; + StringScriptFieldsScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; } - public StringScriptFieldsScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + private final Consumer sync; + + public StringScriptFieldsScript( + Map params, + SourceLookup source, + DocLookup fieldData, + LeafReaderContext ctx, + Consumer sync + ) { super(params, source, fieldData, ctx); + this.sync = sync; } - public abstract String execute(); + /** + * Expose the consumer to the script. + *

+ * This is temporary and I'll remove it in the next PR when I figure out class methods. + */ + public Consumer getSync() { + return sync; + } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java deleted file mode 100644 index 6a3114bd15698..0000000000000 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class LongArrayScriptFieldScriptTests extends ScriptFieldScriptTestCase< - LongArrayScriptFieldScript, - LongArrayScriptFieldScript.Factory, - LongArrayScriptFieldScript.LeafFactory, - Long> { - - public void testConstant() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - }; - assertThat(execute(indexBuilder, "new long[] {10, 100}"), equalTo(List.of(10L, 100L, 10L, 100L))); - } - - public void testSource() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100, \"bar\": 200}")))); - }; - assertThat(execute(indexBuilder, "new long[] {source['foo'], source['bar']}"), equalTo(List.of(10L, 20L, 100L, 200L))); - } - - public void testDocValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100), new SortedNumericDocValuesField("foo", 200))); - }; - assertThat( - execute(indexBuilder, "def foo = doc['foo']; new long[] {foo[0], foo[1]}", new NumberFieldType("foo", NumberType.LONG)), - equalTo(List.of(10L, 20L, 100L, 200L)) - ); - } - - @Override - protected ScriptContext scriptContext() { - return LongArrayScriptFieldScript.CONTEXT; - } - - @Override - protected LongArrayScriptFieldScript.LeafFactory newLeafFactory( - LongArrayScriptFieldScript.Factory factory, - Map params, - SourceLookup source, - DocLookup fieldData - ) { - return factory.newFactory(params, source, fieldData); - } - - @Override - protected LongArrayScriptFieldScript newInstance(LongArrayScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) - throws IOException { - return leafFactory.newInstance(context); - } - - @Override - protected void collect(LongArrayScriptFieldScript script, List result) { - for (long l : script.execute()) { - result.add(l); - } - } -} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index e10a5c822f07a..72f101a81dfa6 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -29,12 +29,21 @@ public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< LongScriptFieldScript.Factory, LongScriptFieldScript.LeafFactory, Long> { + public void testConstant() throws IOException { CheckedConsumer indexBuilder = iw -> { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); }; - assertThat(execute(indexBuilder, "10"), equalTo(List.of(10L, 10L))); + assertThat(execute(indexBuilder, "sync.accept(10)"), equalTo(List.of(10L, 10L))); + } + + public void testTwoConstants() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + }; + assertThat(execute(indexBuilder, "sync.accept(10); sync.accept(20)"), equalTo(List.of(10L, 20L, 10L, 20L))); } public void testSource() throws IOException { @@ -42,7 +51,15 @@ public void testSource() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100}")))); }; - assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of(10L, 100L))); + assertThat(execute(indexBuilder, "sync.accept(source['foo'])"), equalTo(List.of(10L, 100L))); + } + + public void testTwoSourceFields() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100, \"bar\": 200}")))); + }; + assertThat(execute(indexBuilder, "sync.accept(source['foo']); sync.accept(source['bar'])"), equalTo(List.of(10L, 20L, 100L, 200L))); } public void testDocValues() throws IOException { @@ -50,7 +67,21 @@ public void testDocValues() throws IOException { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100))); }; - assertThat(execute(indexBuilder, "doc['foo'].value", new NumberFieldType("foo", NumberType.LONG)), equalTo(List.of(10L, 100L))); + assertThat( + execute(indexBuilder, "sync.accept(doc['foo'].value)", new NumberFieldType("foo", NumberType.LONG)), + equalTo(List.of(10L, 100L)) + ); + } + + public void testTwoDocValuesValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100), new SortedNumericDocValuesField("foo", 200))); + }; + assertThat( + execute(indexBuilder, "def foo = doc['foo']; sync.accept(foo[0]); sync.accept(foo[1])", new NumberFieldType("foo", NumberType.LONG)), + equalTo(List.of(10L, 20L, 100L, 200L)) + ); } @Override @@ -69,13 +100,9 @@ protected LongScriptFieldScript.LeafFactory newLeafFactory( } @Override - protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) + protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context, List result) throws IOException { - return leafFactory.newInstance(context); - } - @Override - protected void collect(LongScriptFieldScript script, List result) { - result.add(script.execute()); + return leafFactory.newInstance(context, result::add); } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 5285f2af8fe65..d54a94facf5b4 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -49,9 +49,7 @@ public abstract class ScriptFieldScriptTestCase params, SourceLookup source, DocLookup fieldData); - protected abstract S newInstance(LF leafFactory, LeafReaderContext context) throws IOException; - - protected abstract void collect(S script, List result); + protected abstract S newInstance(LF leafFactory, LeafReaderContext context, List results) throws IOException; protected final List execute(CheckedConsumer indexBuilder, String script, MappedFieldType... types) throws IOException { @@ -84,7 +82,7 @@ public ScoreMode scoreMode() { @Override public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - S compiled = newInstance(leafFactory, context); + S compiled = newInstance(leafFactory, context, result); return new LeafCollector() { @Override public void setScorer(Scorable scorer) throws IOException {} @@ -92,7 +90,7 @@ public void setScorer(Scorable scorer) throws IOException {} @Override public void collect(int doc) throws IOException { compiled.setDocument(doc); - ScriptFieldScriptTestCase.this.collect(compiled, result); + compiled.execute(); } }; } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java index 55198ce968469..0694ec3616835 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java @@ -34,7 +34,15 @@ public void testConstant() throws IOException { iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); }; - assertThat(execute(indexBuilder, "\"cat\""), equalTo(List.of("cat", "cat"))); + assertThat(execute(indexBuilder, "sync.accept(\"cat\")"), equalTo(List.of("cat", "cat"))); + } + + public void testTwoConstants() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + }; + assertThat(execute(indexBuilder, "sync.accept(\"cat\"); sync.accept(\"dog\")"), equalTo(List.of("cat", "dog", "cat", "dog"))); } public void testSource() throws IOException { @@ -42,7 +50,18 @@ public void testSource() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); }; - assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of("cat", "dog"))); + assertThat(execute(indexBuilder, "sync.accept(source['foo'])"), equalTo(List.of("cat", "dog"))); + } + + public void testTwoSourceFields() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\", \"bar\": \"chicken\"}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\", \"bar\": \"pig\"}")))); + }; + assertThat( + execute(indexBuilder, "sync.accept(source['foo']); sync.accept(source['bar'])"), + equalTo(List.of("cat", "chicken", "dog", "pig")) + ); } public void testDocValues() throws IOException { @@ -50,7 +69,25 @@ public void testDocValues() throws IOException { iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); }; - assertThat(execute(indexBuilder, "doc['foo'].value", new KeywordFieldType("foo")), equalTo(List.of("cat", "dog"))); + assertThat(execute(indexBuilder, "sync.accept(doc['foo'].value)", new KeywordFieldType("foo")), equalTo(List.of("cat", "dog"))); + } + + public void testTwoDocValuesValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument( + List.of( + new SortedSetDocValuesField("foo", new BytesRef("cat")), + new SortedSetDocValuesField("foo", new BytesRef("chicken")) + ) + ); + iw.addDocument( + List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")), new SortedSetDocValuesField("foo", new BytesRef("pig"))) + ); + }; + assertThat( + execute(indexBuilder, "def foo = doc['foo']; sync.accept(foo.get(0)); sync.accept(foo.get(1))", new KeywordFieldType("foo")), + equalTo(List.of("cat", "chicken", "dog", "pig")) + ); } @Override @@ -69,12 +106,11 @@ protected StringScriptFieldsScript.LeafFactory newLeafFactory( } @Override - protected StringScriptFieldsScript newInstance(StringScriptFieldsScript.LeafFactory leafFactory, LeafReaderContext context) throws IOException { - return leafFactory.newInstance(context); - } - - @Override - protected void collect(StringScriptFieldsScript script, List result) { - result.add(script.execute()); + protected StringScriptFieldsScript newInstance( + StringScriptFieldsScript.LeafFactory leafFactory, + LeafReaderContext context, + List result + ) throws IOException { + return leafFactory.newInstance(context, result::add); } } From 7668b8d27ce8a7a962f9dd30286ab72a09f761e0 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 18 Jun 2020 13:58:08 -0400 Subject: [PATCH 2/2] Make running script a little cleaner --- x-pack/plugin/runtime-fields/build.gradle | 2 +- .../runtimefields/LongScriptFieldScript.java | 26 +++++++++----- .../xpack/runtimefields/RuntimeFields.java | 2 +- .../RuntimeFieldsPainlessExtension.java | 24 +++++++++++++ ...ript.java => StringScriptFieldScript.java} | 32 +++++++++++------ ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/runtimefields/long_whitelist.txt | 14 ++++++++ .../xpack/runtimefields/string_whitelist.txt | 14 ++++++++ .../LongScriptFieldScriptTests.java | 20 +++++------ .../ScriptFieldScriptTestCase.java | 5 +-- ...java => StringScriptFieldScriptTests.java} | 36 +++++++++---------- 11 files changed, 125 insertions(+), 51 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java rename x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/{StringScriptFieldsScript.java => StringScriptFieldScript.java} (56%) create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt rename x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/{StringScriptFieldsScriptTests.java => StringScriptFieldScriptTests.java} (73%) diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle index b1fbb6b839f63..ebba3ddfffed6 100644 --- a/x-pack/plugin/runtime-fields/build.gradle +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -6,7 +6,7 @@ esplugin { name 'x-pack-runtime-fields' description 'A module which adds support for runtime fields' classname 'org.elasticsearch.xpack.runtimefields.RuntimeFields' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['x-pack-core', 'lang-painless'] } archivesBaseName = 'x-pack-runtime-fields' diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index 43d78f724ef5f..aad1aa45fcac5 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -7,17 +7,24 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.function.LongConsumer; public abstract class LongScriptFieldScript extends AbstractScriptFieldsScript { - public static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt")); + } + public static final String[] PARAMETERS = {}; public interface Factory extends ScriptFactory { @@ -40,12 +47,15 @@ public LongScriptFieldScript( this.sync = sync; } - /** - * Expose the consumer to the script. - *

- * This is temporary and I'll remove it in the next PR when I figure out class methods. - */ - public LongConsumer getSync() { - return sync; + public static class Value { + private final LongScriptFieldScript script; + + public Value(LongScriptFieldScript script) { + this.script = script; + } + + public void value(long v) { + script.sync.accept(v); + } } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 777495c834b46..e15d9b4bc3b18 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -27,7 +27,7 @@ public Map getMappers() { public List> getContexts() { return List.of( LongScriptFieldScript.CONTEXT, - StringScriptFieldsScript.CONTEXT + StringScriptFieldScript.CONTEXT ); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java new file mode 100644 index 0000000000000..5757b6667e29c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.script.ScriptContext; + +import java.util.List; +import java.util.Map; + +public class RuntimeFieldsPainlessExtension implements PainlessExtension { + @Override + public Map, List> getContextWhitelists() { + return Map.of( + LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist(), + StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist() + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java similarity index 56% rename from x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java rename to x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 0e57878196f46..3e709972a1af6 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -7,17 +7,24 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.function.Consumer; -public abstract class StringScriptFieldsScript extends AbstractScriptFieldsScript { - public static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); +public abstract class StringScriptFieldScript extends AbstractScriptFieldsScript { + static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt")); + } + public static final String[] PARAMETERS = {}; public interface Factory extends ScriptFactory { @@ -25,12 +32,12 @@ public interface Factory extends ScriptFactory { } public static interface LeafFactory { - StringScriptFieldsScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; + StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; } private final Consumer sync; - public StringScriptFieldsScript( + public StringScriptFieldScript( Map params, SourceLookup source, DocLookup fieldData, @@ -41,12 +48,15 @@ public StringScriptFieldsScript( this.sync = sync; } - /** - * Expose the consumer to the script. - *

- * This is temporary and I'll remove it in the next PR when I figure out class methods. - */ - public Consumer getSync() { - return sync; + public static class Value { + private final StringScriptFieldScript script; + + public Value(StringScriptFieldScript script) { + this.script = script; + } + + public void value(String v) { + script.sync.accept(v); + } } } diff --git a/x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..cd804b8d8f6c9 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension \ No newline at end of file diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt new file mode 100644 index 0000000000000..630c679bc21cc --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt @@ -0,0 +1,14 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for long-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.LongScriptFieldScript @no_import { +} + +static_import { + void value(org.elasticsearch.xpack.runtimefields.LongScriptFieldScript, long) bound_to org.elasticsearch.xpack.runtimefields.LongScriptFieldScript$Value +} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt new file mode 100644 index 0000000000000..e0c4e367850f3 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt @@ -0,0 +1,14 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for long-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.StringScriptFieldScript @no_import { +} + +static_import { + void value(org.elasticsearch.xpack.runtimefields.StringScriptFieldScript, String) bound_to org.elasticsearch.xpack.runtimefields.StringScriptFieldScript$Value +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 72f101a81dfa6..05d48808b5620 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -35,7 +35,7 @@ public void testConstant() throws IOException { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); }; - assertThat(execute(indexBuilder, "sync.accept(10)"), equalTo(List.of(10L, 10L))); + assertThat(execute(indexBuilder, "value(10)"), equalTo(List.of(10L, 10L))); } public void testTwoConstants() throws IOException { @@ -43,43 +43,43 @@ public void testTwoConstants() throws IOException { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); }; - assertThat(execute(indexBuilder, "sync.accept(10); sync.accept(20)"), equalTo(List.of(10L, 20L, 10L, 20L))); + assertThat(execute(indexBuilder, "value(10); value(20)"), equalTo(List.of(10L, 20L, 10L, 20L))); } public void testSource() throws IOException { CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100}")))); }; - assertThat(execute(indexBuilder, "sync.accept(source['foo'])"), equalTo(List.of(10L, 100L))); + assertThat(execute(indexBuilder, "value(source['foo'] * 10)"), equalTo(List.of(10L, 100L))); } public void testTwoSourceFields() throws IOException { CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1, \"bar\": 2}")))); iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100, \"bar\": 200}")))); }; - assertThat(execute(indexBuilder, "sync.accept(source['foo']); sync.accept(source['bar'])"), equalTo(List.of(10L, 20L, 100L, 200L))); + assertThat(execute(indexBuilder, "value(source['foo'] * 10); value(source['bar'] * 10)"), equalTo(List.of(10L, 20L, 100L, 200L))); } public void testDocValues() throws IOException { CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 1))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100))); }; assertThat( - execute(indexBuilder, "sync.accept(doc['foo'].value)", new NumberFieldType("foo", NumberType.LONG)), + execute(indexBuilder, "value(doc['foo'].value * 10)", new NumberFieldType("foo", NumberType.LONG)), equalTo(List.of(10L, 100L)) ); } public void testTwoDocValuesValues() throws IOException { CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 1), new SortedNumericDocValuesField("foo", 2))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100), new SortedNumericDocValuesField("foo", 200))); }; assertThat( - execute(indexBuilder, "def foo = doc['foo']; sync.accept(foo[0]); sync.accept(foo[1])", new NumberFieldType("foo", NumberType.LONG)), + execute(indexBuilder, "for (long l : doc['foo']) {value(l * 10)}", new NumberFieldType("foo", NumberType.LONG)), equalTo(List.of(10L, 20L, 100L, 200L)) ); } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index d54a94facf5b4..c5c40ffa0972b 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -54,8 +54,9 @@ public abstract class ScriptFieldScriptTestCase execute(CheckedConsumer indexBuilder, String script, MappedFieldType... types) throws IOException { - // TODO replace painless with mock script engine - ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(new PainlessPlugin(), new RuntimeFields())); + PainlessPlugin painlessPlugin = new PainlessPlugin(); + painlessPlugin.reloadSPI(Thread.currentThread().getContextClassLoader()); + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); Map params = new HashMap<>(); SourceLookup source = new SourceLookup(); MapperService mapperService = mock(MapperService.class); diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java similarity index 73% rename from x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java rename to x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 0694ec3616835..658001f743281 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -23,10 +23,10 @@ import static org.hamcrest.Matchers.equalTo; -public class StringScriptFieldsScriptTests extends ScriptFieldScriptTestCase< - StringScriptFieldsScript, - StringScriptFieldsScript.Factory, - StringScriptFieldsScript.LeafFactory, +public class StringScriptFieldScriptTests extends ScriptFieldScriptTestCase< + StringScriptFieldScript, + StringScriptFieldScript.Factory, + StringScriptFieldScript.LeafFactory, String> { public void testConstant() throws IOException { @@ -34,7 +34,7 @@ public void testConstant() throws IOException { iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); }; - assertThat(execute(indexBuilder, "sync.accept(\"cat\")"), equalTo(List.of("cat", "cat"))); + assertThat(execute(indexBuilder, "value('cat')"), equalTo(List.of("cat", "cat"))); } public void testTwoConstants() throws IOException { @@ -42,7 +42,7 @@ public void testTwoConstants() throws IOException { iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); }; - assertThat(execute(indexBuilder, "sync.accept(\"cat\"); sync.accept(\"dog\")"), equalTo(List.of("cat", "dog", "cat", "dog"))); + assertThat(execute(indexBuilder, "value('cat'); value('dog')"), equalTo(List.of("cat", "dog", "cat", "dog"))); } public void testSource() throws IOException { @@ -50,7 +50,7 @@ public void testSource() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); }; - assertThat(execute(indexBuilder, "sync.accept(source['foo'])"), equalTo(List.of("cat", "dog"))); + assertThat(execute(indexBuilder, "value(source['foo'] + 'o')"), equalTo(List.of("cato", "dogo"))); } public void testTwoSourceFields() throws IOException { @@ -59,8 +59,8 @@ public void testTwoSourceFields() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\", \"bar\": \"pig\"}")))); }; assertThat( - execute(indexBuilder, "sync.accept(source['foo']); sync.accept(source['bar'])"), - equalTo(List.of("cat", "chicken", "dog", "pig")) + execute(indexBuilder, "value(source['foo'] + 'o'); value(source['bar'] + 'ie')"), + equalTo(List.of("cato", "chickenie", "dogo", "pigie")) ); } @@ -69,7 +69,7 @@ public void testDocValues() throws IOException { iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); }; - assertThat(execute(indexBuilder, "sync.accept(doc['foo'].value)", new KeywordFieldType("foo")), equalTo(List.of("cat", "dog"))); + assertThat(execute(indexBuilder, "value(doc['foo'].value + 'o')", new KeywordFieldType("foo")), equalTo(List.of("cato", "dogo"))); } public void testTwoDocValuesValues() throws IOException { @@ -85,19 +85,19 @@ public void testTwoDocValuesValues() throws IOException { ); }; assertThat( - execute(indexBuilder, "def foo = doc['foo']; sync.accept(foo.get(0)); sync.accept(foo.get(1))", new KeywordFieldType("foo")), - equalTo(List.of("cat", "chicken", "dog", "pig")) + execute(indexBuilder, "for (String s: doc['foo']) {value(s + 'o')}", new KeywordFieldType("foo")), + equalTo(List.of("cato", "chickeno", "dogo", "pigo")) ); } @Override - protected ScriptContext scriptContext() { - return StringScriptFieldsScript.CONTEXT; + protected ScriptContext scriptContext() { + return StringScriptFieldScript.CONTEXT; } @Override - protected StringScriptFieldsScript.LeafFactory newLeafFactory( - StringScriptFieldsScript.Factory factory, + protected StringScriptFieldScript.LeafFactory newLeafFactory( + StringScriptFieldScript.Factory factory, Map params, SourceLookup source, DocLookup fieldData @@ -106,8 +106,8 @@ protected StringScriptFieldsScript.LeafFactory newLeafFactory( } @Override - protected StringScriptFieldsScript newInstance( - StringScriptFieldsScript.LeafFactory leafFactory, + protected StringScriptFieldScript newInstance( + StringScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context, List result ) throws IOException {