From ecbed2c2b868bb59f679f03530aa3f423dbadf2a Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 17 Jun 2020 16:10:15 +0200 Subject: [PATCH 01/82] Add runtime fields plugin under x-pack --- x-pack/plugin/runtime-fields/build.gradle | 26 ++++++++++++++++ .../xpack/runtimefields/RuntimeFields.java | 30 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 x-pack/plugin/runtime-fields/build.gradle create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle new file mode 100644 index 0000000000000..4b9ae1edb7d93 --- /dev/null +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -0,0 +1,26 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'x-pack-runtime-fields' + description 'A module which adds support for runtime fields' + classname 'org.elasticsearch.xpack.runtimefields.RuntimeFields' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-runtime-fields' + +compileJava.options.compilerArgs << "-Xlint:-rawtypes" +compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" + +dependencies { + compileOnly project(":server") + + compileOnly project(path: xpackModule('core'), configuration: 'default') +} + +dependencyLicenses { + ignoreSha 'x-pack-core' +} + +integTest.enabled = false diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java new file mode 100644 index 0000000000000..2f3bf46bf3c47 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { + + @Override + public Map getMappers() { + return Collections.emptyMap(); + } + + @Override + public List> getContexts() { + return Collections.emptyList(); + } +} From 7ab427997fcc25ecb91d033a33d5206d1b094098 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 17 Jun 2020 15:37:31 -0400 Subject: [PATCH 02/82] Hack together some script contexts Put together some example script contexts for scripted fields. This is almost certainly wrong, but it gets us something to think about. --- .../search/lookup/DocLookup.java | 2 +- x-pack/plugin/runtime-fields/build.gradle | 2 +- .../AbstractScriptFieldsScript.java | 63 ++++++++++ .../LongArrayScriptFieldScript.java | 34 +++++ .../runtimefields/LongScriptFieldScript.java | 34 +++++ .../xpack/runtimefields/RuntimeFields.java | 6 +- .../StringScriptFieldsScript.java | 34 +++++ .../LongArrayScriptFieldScriptTests.java | 87 +++++++++++++ .../LongScriptFieldScriptTests.java | 81 ++++++++++++ .../ScriptFieldScriptTestCase.java | 117 ++++++++++++++++++ .../StringScriptFieldsScriptTests.java | 80 ++++++++++++ 11 files changed, 537 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java diff --git a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java index 0022cdfdc9d99..1f0e53fca14c9 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java @@ -30,7 +30,7 @@ public class DocLookup { private final MapperService mapperService; private final Function> fieldDataLookup; - DocLookup(MapperService mapperService, Function> fieldDataLookup) { + public DocLookup(MapperService mapperService, Function> fieldDataLookup) { this.mapperService = mapperService; this.fieldDataLookup = fieldDataLookup; } diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle index 4b9ae1edb7d93..b1fbb6b839f63 100644 --- a/x-pack/plugin/runtime-fields/build.gradle +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -15,7 +15,7 @@ compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" dependencies { compileOnly project(":server") - + compileOnly project(':modules:lang-painless') // TODO we don't want to depend on painless if we're not extending it. But it is convenient for testing now. compileOnly project(path: xpackModule('core'), configuration: 'default') } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java new file mode 100644 index 0000000000000..95c419b6ad558 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.LeafDocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.util.Map; + +/** + * Abstract base for scripts to execute to build scripted fields. Inspired by + * {@link AggregationScript} but hopefully with less historical baggage. + */ +public abstract class AbstractScriptFieldsScript { + private final Map params; + private final LeafReaderContext ctx; + private final SourceLookup source; + private final LeafDocLookup fieldData; + + public AbstractScriptFieldsScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + this.params = params; + this.source = source; + this.fieldData = fieldData.getLeafDocLookup(ctx); + this.ctx = ctx; + } + + /** + * Set the document to run the script against. + */ + public void setDocument(int docId) { + source.setSegmentAndDocument(ctx, docId); + fieldData.setDocument(docId); + } + + /** + * Expose the {@code params} of the script to the script itself. + */ + public Map getParams() { + return params; + } + + /** + * Expose the {@code _source} to the script. + */ + public Map getSource() { + return source; + } + + /** + * Expose field data to the script as {@code doc}. + */ + public Map> getDoc() { + return fieldData; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java new file mode 100644 index 0000000000000..e9676f580b45f --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Map; + +public abstract class LongArrayScriptFieldScript extends AbstractScriptFieldsScript { + public static final ScriptContext CONTEXT = new ScriptContext<>("long_array_script_field", Factory.class); + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + public static interface LeafFactory { + LongArrayScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + public LongArrayScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + super(params, source, fieldData, ctx); + } + + public abstract long[] execute(); +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java new file mode 100644 index 0000000000000..4e1dce5c9facc --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Map; + +public abstract class LongScriptFieldScript extends AbstractScriptFieldsScript { + public static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + public static interface LeafFactory { + LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + public LongScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + super(params, source, fieldData, ctx); + } + + public abstract long execute(); +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 2f3bf46bf3c47..0a91bd6db2021 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -25,6 +25,10 @@ public Map getMappers() { @Override public List> getContexts() { - return Collections.emptyList(); + return List.of( + LongArrayScriptFieldScript.CONTEXT, + LongScriptFieldScript.CONTEXT, + StringScriptFieldsScript.CONTEXT + ); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java new file mode 100644 index 0000000000000..b8f3809de3af3 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Map; + +public abstract class StringScriptFieldsScript extends AbstractScriptFieldsScript { + public static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + public static interface LeafFactory { + StringScriptFieldsScript newInstance(LeafReaderContext ctx) throws IOException; + } + + public StringScriptFieldsScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + super(params, source, fieldData, ctx); + } + + public abstract String execute(); +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java new file mode 100644 index 0000000000000..6a3114bd15698 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class LongArrayScriptFieldScriptTests extends ScriptFieldScriptTestCase< + LongArrayScriptFieldScript, + LongArrayScriptFieldScript.Factory, + LongArrayScriptFieldScript.LeafFactory, + Long> { + + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + }; + assertThat(execute(indexBuilder, "new long[] {10, 100}"), equalTo(List.of(10L, 100L, 10L, 100L))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100, \"bar\": 200}")))); + }; + assertThat(execute(indexBuilder, "new long[] {source['foo'], source['bar']}"), equalTo(List.of(10L, 20L, 100L, 200L))); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100), new SortedNumericDocValuesField("foo", 200))); + }; + assertThat( + execute(indexBuilder, "def foo = doc['foo']; new long[] {foo[0], foo[1]}", new NumberFieldType("foo", NumberType.LONG)), + equalTo(List.of(10L, 20L, 100L, 200L)) + ); + } + + @Override + protected ScriptContext scriptContext() { + return LongArrayScriptFieldScript.CONTEXT; + } + + @Override + protected LongArrayScriptFieldScript.LeafFactory newLeafFactory( + LongArrayScriptFieldScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected LongArrayScriptFieldScript newInstance(LongArrayScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) + throws IOException { + return leafFactory.newInstance(context); + } + + @Override + protected void collect(LongArrayScriptFieldScript script, List result) { + for (long l : script.execute()) { + result.add(l); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java new file mode 100644 index 0000000000000..e10a5c822f07a --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< + LongScriptFieldScript, + LongScriptFieldScript.Factory, + LongScriptFieldScript.LeafFactory, + Long> { + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + }; + assertThat(execute(indexBuilder, "10"), equalTo(List.of(10L, 10L))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100}")))); + }; + assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of(10L, 100L))); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100))); + }; + assertThat(execute(indexBuilder, "doc['foo'].value", new NumberFieldType("foo", NumberType.LONG)), equalTo(List.of(10L, 100L))); + } + + @Override + protected ScriptContext scriptContext() { + return LongScriptFieldScript.CONTEXT; + } + + @Override + protected LongScriptFieldScript.LeafFactory newLeafFactory( + LongScriptFieldScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) + throws IOException { + return leafFactory.newInstance(context); + } + + @Override + protected void collect(LongScriptFieldScript script, List result) { + result.add(script.execute()); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java new file mode 100644 index 0000000000000..5285f2af8fe65 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.store.Directory; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.painless.PainlessPlugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public abstract class ScriptFieldScriptTestCase extends ESTestCase { + protected abstract ScriptContext scriptContext(); + + protected abstract LF newLeafFactory(F factory, Map params, SourceLookup source, DocLookup fieldData); + + protected abstract S newInstance(LF leafFactory, LeafReaderContext context) throws IOException; + + protected abstract void collect(S script, List result); + + protected final List execute(CheckedConsumer indexBuilder, String script, MappedFieldType... types) + throws IOException { + + // TODO replace painless with mock script engine + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(new PainlessPlugin(), new RuntimeFields())); + Map params = new HashMap<>(); + SourceLookup source = new SourceLookup(); + MapperService mapperService = mock(MapperService.class); + for (MappedFieldType type : types) { + when(mapperService.fieldType(type.name())).thenReturn(type); + } + Function> fieldDataLookup = ft -> ft.fielddataBuilder("test") + .build(indexSettings(), ft, null, new NoneCircuitBreakerService(), mapperService); + DocLookup fieldData = new DocLookup(mapperService, fieldDataLookup); + try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { + F factory = scriptService.compile(new Script(script), scriptContext()); + + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexBuilder.accept(indexWriter); + try (DirectoryReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newSearcher(reader); + LF leafFactory = newLeafFactory(factory, params, source, fieldData); + List result = new ArrayList<>(); + searcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + S compiled = newInstance(leafFactory, context); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + compiled.setDocument(doc); + ScriptFieldScriptTestCase.this.collect(compiled, result); + } + }; + } + }); + return result; + } + } + } + } + + private IndexSettings indexSettings() { + return new IndexSettings( + IndexMetadata.builder("_index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(), + Settings.EMPTY + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java new file mode 100644 index 0000000000000..55198ce968469 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class StringScriptFieldsScriptTests extends ScriptFieldScriptTestCase< + StringScriptFieldsScript, + StringScriptFieldsScript.Factory, + StringScriptFieldsScript.LeafFactory, + String> { + + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + }; + assertThat(execute(indexBuilder, "\"cat\""), equalTo(List.of("cat", "cat"))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); + }; + assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of("cat", "dog"))); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); + }; + assertThat(execute(indexBuilder, "doc['foo'].value", new KeywordFieldType("foo")), equalTo(List.of("cat", "dog"))); + } + + @Override + protected ScriptContext scriptContext() { + return StringScriptFieldsScript.CONTEXT; + } + + @Override + protected StringScriptFieldsScript.LeafFactory newLeafFactory( + StringScriptFieldsScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected StringScriptFieldsScript newInstance(StringScriptFieldsScript.LeafFactory leafFactory, LeafReaderContext context) throws IOException { + return leafFactory.newInstance(context); + } + + @Override + protected void collect(StringScriptFieldsScript script, List result) { + result.add(script.execute()); + } +} From c6472787964658940f3a2800e9b3f918531a706d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 18 Jun 2020 16:20:40 -0400 Subject: [PATCH 03/82] Scripted fields: rework script contexts (#58342) This reworks the script contexts to support multiple values for each hit. Just like before, these are almost certainly not the final implementation, but they give us something to iterate on. --- x-pack/plugin/runtime-fields/build.gradle | 2 +- .../AbstractScriptFieldsScript.java | 11 +- .../LongArrayScriptFieldScript.java | 34 ----- .../runtimefields/LongScriptFieldScript.java | 35 +++++- .../xpack/runtimefields/RuntimeFields.java | 3 +- .../RuntimeFieldsPainlessExtension.java | 24 ++++ .../StringScriptFieldScript.java | 62 ++++++++++ .../StringScriptFieldsScript.java | 34 ----- ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/runtimefields/long_whitelist.txt | 14 +++ .../xpack/runtimefields/string_whitelist.txt | 14 +++ .../LongArrayScriptFieldScriptTests.java | 87 ------------- .../LongScriptFieldScriptTests.java | 49 ++++++-- .../ScriptFieldScriptTestCase.java | 13 +- .../StringScriptFieldScriptTests.java | 116 ++++++++++++++++++ .../StringScriptFieldsScriptTests.java | 80 ------------ 16 files changed, 315 insertions(+), 264 deletions(-) delete mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java delete mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt delete mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java delete mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle index b1fbb6b839f63..ebba3ddfffed6 100644 --- a/x-pack/plugin/runtime-fields/build.gradle +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -6,7 +6,7 @@ esplugin { name 'x-pack-runtime-fields' description 'A module which adds support for runtime fields' classname 'org.elasticsearch.xpack.runtimefields.RuntimeFields' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['x-pack-core', 'lang-painless'] } archivesBaseName = 'x-pack-runtime-fields' diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java index 95c419b6ad558..952192c52dbf7 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java @@ -35,7 +35,7 @@ public AbstractScriptFieldsScript(Map params, SourceLookup sourc /** * Set the document to run the script against. */ - public void setDocument(int docId) { + public final void setDocument(int docId) { source.setSegmentAndDocument(ctx, docId); fieldData.setDocument(docId); } @@ -43,21 +43,24 @@ public void setDocument(int docId) { /** * Expose the {@code params} of the script to the script itself. */ - public Map getParams() { + public final Map getParams() { return params; } /** * Expose the {@code _source} to the script. */ - public Map getSource() { + public final Map getSource() { return source; } /** * Expose field data to the script as {@code doc}. */ - public Map> getDoc() { + public final Map> getDoc() { return fieldData; } + + public abstract void execute(); + } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java deleted file mode 100644 index e9676f580b45f..0000000000000 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptFactory; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.Map; - -public abstract class LongArrayScriptFieldScript extends AbstractScriptFieldsScript { - public static final ScriptContext CONTEXT = new ScriptContext<>("long_array_script_field", Factory.class); - public static final String[] PARAMETERS = {}; - - public interface Factory extends ScriptFactory { - LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); - } - public static interface LeafFactory { - LongArrayScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; - } - - public LongArrayScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { - super(params, source, fieldData, ctx); - } - - public abstract long[] execute(); -} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index 4e1dce5c9facc..aad1aa45fcac5 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -7,28 +7,55 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.function.LongConsumer; public abstract class LongScriptFieldScript extends AbstractScriptFieldsScript { - public static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt")); + } + public static final String[] PARAMETERS = {}; public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } public static interface LeafFactory { - LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException; } - public LongScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + private final LongConsumer sync; + + public LongScriptFieldScript( + Map params, + SourceLookup source, + DocLookup fieldData, + LeafReaderContext ctx, + LongConsumer sync + ) { super(params, source, fieldData, ctx); + this.sync = sync; } - public abstract long execute(); + public static class Value { + private final LongScriptFieldScript script; + + public Value(LongScriptFieldScript script) { + this.script = script; + } + + public void value(long v) { + script.sync.accept(v); + } + } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 0a91bd6db2021..e15d9b4bc3b18 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -26,9 +26,8 @@ public Map getMappers() { @Override public List> getContexts() { return List.of( - LongArrayScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT, - StringScriptFieldsScript.CONTEXT + StringScriptFieldScript.CONTEXT ); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java new file mode 100644 index 0000000000000..5757b6667e29c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.script.ScriptContext; + +import java.util.List; +import java.util.Map; + +public class RuntimeFieldsPainlessExtension implements PainlessExtension { + @Override + public Map, List> getContextWhitelists() { + return Map.of( + LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist(), + StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist() + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java new file mode 100644 index 0000000000000..3e709972a1af6 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +public abstract class StringScriptFieldScript extends AbstractScriptFieldsScript { + static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + + public static interface LeafFactory { + StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; + } + + private final Consumer sync; + + public StringScriptFieldScript( + Map params, + SourceLookup source, + DocLookup fieldData, + LeafReaderContext ctx, + Consumer sync + ) { + super(params, source, fieldData, ctx); + this.sync = sync; + } + + public static class Value { + private final StringScriptFieldScript script; + + public Value(StringScriptFieldScript script) { + this.script = script; + } + + public void value(String v) { + script.sync.accept(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java deleted file mode 100644 index b8f3809de3af3..0000000000000 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptFactory; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.Map; - -public abstract class StringScriptFieldsScript extends AbstractScriptFieldsScript { - public static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); - public static final String[] PARAMETERS = {}; - - public interface Factory extends ScriptFactory { - LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); - } - public static interface LeafFactory { - StringScriptFieldsScript newInstance(LeafReaderContext ctx) throws IOException; - } - - public StringScriptFieldsScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { - super(params, source, fieldData, ctx); - } - - public abstract String execute(); -} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..cd804b8d8f6c9 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension \ No newline at end of file diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt new file mode 100644 index 0000000000000..630c679bc21cc --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt @@ -0,0 +1,14 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for long-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.LongScriptFieldScript @no_import { +} + +static_import { + void value(org.elasticsearch.xpack.runtimefields.LongScriptFieldScript, long) bound_to org.elasticsearch.xpack.runtimefields.LongScriptFieldScript$Value +} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt new file mode 100644 index 0000000000000..e0c4e367850f3 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt @@ -0,0 +1,14 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for long-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.StringScriptFieldScript @no_import { +} + +static_import { + void value(org.elasticsearch.xpack.runtimefields.StringScriptFieldScript, String) bound_to org.elasticsearch.xpack.runtimefields.StringScriptFieldScript$Value +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java deleted file mode 100644 index 6a3114bd15698..0000000000000 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class LongArrayScriptFieldScriptTests extends ScriptFieldScriptTestCase< - LongArrayScriptFieldScript, - LongArrayScriptFieldScript.Factory, - LongArrayScriptFieldScript.LeafFactory, - Long> { - - public void testConstant() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - }; - assertThat(execute(indexBuilder, "new long[] {10, 100}"), equalTo(List.of(10L, 100L, 10L, 100L))); - } - - public void testSource() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100, \"bar\": 200}")))); - }; - assertThat(execute(indexBuilder, "new long[] {source['foo'], source['bar']}"), equalTo(List.of(10L, 20L, 100L, 200L))); - } - - public void testDocValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100), new SortedNumericDocValuesField("foo", 200))); - }; - assertThat( - execute(indexBuilder, "def foo = doc['foo']; new long[] {foo[0], foo[1]}", new NumberFieldType("foo", NumberType.LONG)), - equalTo(List.of(10L, 20L, 100L, 200L)) - ); - } - - @Override - protected ScriptContext scriptContext() { - return LongArrayScriptFieldScript.CONTEXT; - } - - @Override - protected LongArrayScriptFieldScript.LeafFactory newLeafFactory( - LongArrayScriptFieldScript.Factory factory, - Map params, - SourceLookup source, - DocLookup fieldData - ) { - return factory.newFactory(params, source, fieldData); - } - - @Override - protected LongArrayScriptFieldScript newInstance(LongArrayScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) - throws IOException { - return leafFactory.newInstance(context); - } - - @Override - protected void collect(LongArrayScriptFieldScript script, List result) { - for (long l : script.execute()) { - result.add(l); - } - } -} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index e10a5c822f07a..05d48808b5620 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -29,28 +29,59 @@ public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< LongScriptFieldScript.Factory, LongScriptFieldScript.LeafFactory, Long> { + public void testConstant() throws IOException { CheckedConsumer indexBuilder = iw -> { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); }; - assertThat(execute(indexBuilder, "10"), equalTo(List.of(10L, 10L))); + assertThat(execute(indexBuilder, "value(10)"), equalTo(List.of(10L, 10L))); + } + + public void testTwoConstants() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + }; + assertThat(execute(indexBuilder, "value(10); value(20)"), equalTo(List.of(10L, 20L, 10L, 20L))); } public void testSource() throws IOException { CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100}")))); }; - assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of(10L, 100L))); + assertThat(execute(indexBuilder, "value(source['foo'] * 10)"), equalTo(List.of(10L, 100L))); + } + + public void testTwoSourceFields() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1, \"bar\": 2}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); + }; + assertThat(execute(indexBuilder, "value(source['foo'] * 10); value(source['bar'] * 10)"), equalTo(List.of(10L, 20L, 100L, 200L))); } public void testDocValues() throws IOException { CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 1))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100))); }; - assertThat(execute(indexBuilder, "doc['foo'].value", new NumberFieldType("foo", NumberType.LONG)), equalTo(List.of(10L, 100L))); + assertThat( + execute(indexBuilder, "value(doc['foo'].value * 10)", new NumberFieldType("foo", NumberType.LONG)), + equalTo(List.of(10L, 100L)) + ); + } + + public void testTwoDocValuesValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 1), new SortedNumericDocValuesField("foo", 2))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); + }; + assertThat( + execute(indexBuilder, "for (long l : doc['foo']) {value(l * 10)}", new NumberFieldType("foo", NumberType.LONG)), + equalTo(List.of(10L, 20L, 100L, 200L)) + ); } @Override @@ -69,13 +100,9 @@ protected LongScriptFieldScript.LeafFactory newLeafFactory( } @Override - protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) + protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context, List result) throws IOException { - return leafFactory.newInstance(context); - } - @Override - protected void collect(LongScriptFieldScript script, List result) { - result.add(script.execute()); + return leafFactory.newInstance(context, result::add); } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 5285f2af8fe65..c5c40ffa0972b 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -49,15 +49,14 @@ public abstract class ScriptFieldScriptTestCase params, SourceLookup source, DocLookup fieldData); - protected abstract S newInstance(LF leafFactory, LeafReaderContext context) throws IOException; - - protected abstract void collect(S script, List result); + protected abstract S newInstance(LF leafFactory, LeafReaderContext context, List results) throws IOException; protected final List execute(CheckedConsumer indexBuilder, String script, MappedFieldType... types) throws IOException { - // TODO replace painless with mock script engine - ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(new PainlessPlugin(), new RuntimeFields())); + PainlessPlugin painlessPlugin = new PainlessPlugin(); + painlessPlugin.reloadSPI(Thread.currentThread().getContextClassLoader()); + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); Map params = new HashMap<>(); SourceLookup source = new SourceLookup(); MapperService mapperService = mock(MapperService.class); @@ -84,7 +83,7 @@ public ScoreMode scoreMode() { @Override public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - S compiled = newInstance(leafFactory, context); + S compiled = newInstance(leafFactory, context, result); return new LeafCollector() { @Override public void setScorer(Scorable scorer) throws IOException {} @@ -92,7 +91,7 @@ public void setScorer(Scorable scorer) throws IOException {} @Override public void collect(int doc) throws IOException { compiled.setDocument(doc); - ScriptFieldScriptTestCase.this.collect(compiled, result); + compiled.execute(); } }; } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java new file mode 100644 index 0000000000000..658001f743281 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class StringScriptFieldScriptTests extends ScriptFieldScriptTestCase< + StringScriptFieldScript, + StringScriptFieldScript.Factory, + StringScriptFieldScript.LeafFactory, + String> { + + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + }; + assertThat(execute(indexBuilder, "value('cat')"), equalTo(List.of("cat", "cat"))); + } + + public void testTwoConstants() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + }; + assertThat(execute(indexBuilder, "value('cat'); value('dog')"), equalTo(List.of("cat", "dog", "cat", "dog"))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); + }; + assertThat(execute(indexBuilder, "value(source['foo'] + 'o')"), equalTo(List.of("cato", "dogo"))); + } + + public void testTwoSourceFields() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\", \"bar\": \"chicken\"}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\", \"bar\": \"pig\"}")))); + }; + assertThat( + execute(indexBuilder, "value(source['foo'] + 'o'); value(source['bar'] + 'ie')"), + equalTo(List.of("cato", "chickenie", "dogo", "pigie")) + ); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); + }; + assertThat(execute(indexBuilder, "value(doc['foo'].value + 'o')", new KeywordFieldType("foo")), equalTo(List.of("cato", "dogo"))); + } + + public void testTwoDocValuesValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument( + List.of( + new SortedSetDocValuesField("foo", new BytesRef("cat")), + new SortedSetDocValuesField("foo", new BytesRef("chicken")) + ) + ); + iw.addDocument( + List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")), new SortedSetDocValuesField("foo", new BytesRef("pig"))) + ); + }; + assertThat( + execute(indexBuilder, "for (String s: doc['foo']) {value(s + 'o')}", new KeywordFieldType("foo")), + equalTo(List.of("cato", "chickeno", "dogo", "pigo")) + ); + } + + @Override + protected ScriptContext scriptContext() { + return StringScriptFieldScript.CONTEXT; + } + + @Override + protected StringScriptFieldScript.LeafFactory newLeafFactory( + StringScriptFieldScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected StringScriptFieldScript newInstance( + StringScriptFieldScript.LeafFactory leafFactory, + LeafReaderContext context, + List result + ) throws IOException { + return leafFactory.newInstance(context, result::add); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java deleted file mode 100644 index 55198ce968469..0000000000000 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.document.SortedSetDocValuesField; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class StringScriptFieldsScriptTests extends ScriptFieldScriptTestCase< - StringScriptFieldsScript, - StringScriptFieldsScript.Factory, - StringScriptFieldsScript.LeafFactory, - String> { - - public void testConstant() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); - }; - assertThat(execute(indexBuilder, "\"cat\""), equalTo(List.of("cat", "cat"))); - } - - public void testSource() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); - }; - assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of("cat", "dog"))); - } - - public void testDocValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); - }; - assertThat(execute(indexBuilder, "doc['foo'].value", new KeywordFieldType("foo")), equalTo(List.of("cat", "dog"))); - } - - @Override - protected ScriptContext scriptContext() { - return StringScriptFieldsScript.CONTEXT; - } - - @Override - protected StringScriptFieldsScript.LeafFactory newLeafFactory( - StringScriptFieldsScript.Factory factory, - Map params, - SourceLookup source, - DocLookup fieldData - ) { - return factory.newFactory(params, source, fieldData); - } - - @Override - protected StringScriptFieldsScript newInstance(StringScriptFieldsScript.LeafFactory leafFactory, LeafReaderContext context) throws IOException { - return leafFactory.newInstance(context); - } - - @Override - protected void collect(StringScriptFieldsScript script, List result) { - result.add(script.execute()); - } -} From 7d3597b4f2b94852ec5f5d9f99dd74d06dd2c22f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 18 Jun 2020 15:29:00 -0400 Subject: [PATCH 04/82] Spotless and maybe tests? --- ...pt.java => AbstractScriptFieldScript.java} | 4 +- .../DoubleScriptFieldScript.java | 63 +++++++++ .../runtimefields/LongScriptFieldScript.java | 6 +- .../xpack/runtimefields/RuntimeFields.java | 5 +- .../RuntimeFieldsPainlessExtension.java | 7 +- .../StringScriptFieldScript.java | 5 +- .../xpack/runtimefields/double_whitelist.txt | 14 ++ .../DoubleScriptFieldScriptTests.java | 124 ++++++++++++++++++ .../ScriptFieldScriptTestCase.java | 8 +- 9 files changed, 221 insertions(+), 15 deletions(-) rename x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/{AbstractScriptFieldsScript.java => AbstractScriptFieldScript.java} (90%) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java similarity index 90% rename from x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java rename to x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java index 952192c52dbf7..b9d605c796530 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -19,13 +19,13 @@ * Abstract base for scripts to execute to build scripted fields. Inspired by * {@link AggregationScript} but hopefully with less historical baggage. */ -public abstract class AbstractScriptFieldsScript { +public abstract class AbstractScriptFieldScript { private final Map params; private final LeafReaderContext ctx; private final SourceLookup source; private final LeafDocLookup fieldData; - public AbstractScriptFieldsScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + public AbstractScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { this.params = params; this.source = source; this.fieldData = fieldData.getLeafDocLookup(ctx); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java new file mode 100644 index 0000000000000..e86bfb20f71b6 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.DoubleConsumer; + +public abstract class DoubleScriptFieldScript extends AbstractScriptFieldScript { + static final ScriptContext CONTEXT = new ScriptContext<>("double_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "double_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + + public interface LeafFactory { + DoubleScriptFieldScript newInstance(LeafReaderContext ctx, DoubleConsumer sync) throws IOException; + } + + private final DoubleConsumer sync; + + public DoubleScriptFieldScript( + Map params, + SourceLookup source, + DocLookup fieldData, + LeafReaderContext ctx, + DoubleConsumer sync + ) { + super(params, source, fieldData, ctx); + this.sync = sync; + } + + public static class Value { + private final DoubleScriptFieldScript script; + + public Value(DoubleScriptFieldScript script) { + this.script = script; + } + + public void value(double v) { + script.sync.accept(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index aad1aa45fcac5..e99fa4a944ba3 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -19,8 +19,9 @@ import java.util.Map; import java.util.function.LongConsumer; -public abstract class LongScriptFieldScript extends AbstractScriptFieldsScript { +public abstract class LongScriptFieldScript extends AbstractScriptFieldScript { static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + static List whitelist() { return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt")); } @@ -30,7 +31,8 @@ static List whitelist() { public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } - public static interface LeafFactory { + + public interface LeafFactory { LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException; } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index e15d9b4bc3b18..6b97b0dcb795c 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -25,9 +25,6 @@ public Map getMappers() { @Override public List> getContexts() { - return List.of( - LongScriptFieldScript.CONTEXT, - StringScriptFieldScript.CONTEXT - ); + return List.of(DoubleScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT, StringScriptFieldScript.CONTEXT); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java index 5757b6667e29c..0ccbeee8069a1 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java @@ -16,9 +16,10 @@ public class RuntimeFieldsPainlessExtension implements PainlessExtension { @Override public Map, List> getContextWhitelists() { - return Map.of( - LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist(), - StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist() + return Map.ofEntries( + Map.entry(DoubleScriptFieldScript.CONTEXT, DoubleScriptFieldScript.whitelist()), + Map.entry(LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist()), + Map.entry(StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist()) ); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 3e709972a1af6..2556381981236 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -19,8 +19,9 @@ import java.util.Map; import java.util.function.Consumer; -public abstract class StringScriptFieldScript extends AbstractScriptFieldsScript { +public abstract class StringScriptFieldScript extends AbstractScriptFieldScript { static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); + static List whitelist() { return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt")); } @@ -31,7 +32,7 @@ public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } - public static interface LeafFactory { + public interface LeafFactory { StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; } diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt new file mode 100644 index 0000000000000..55a177f21579c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt @@ -0,0 +1,14 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for long-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript @no_import { +} + +static_import { + void value(org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript, double) bound_to org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript$Value +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java new file mode 100644 index 0000000000000..37c6c5d35ce41 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class DoubleScriptFieldScriptTests extends ScriptFieldScriptTestCase< + DoubleScriptFieldScript, + DoubleScriptFieldScript.Factory, + DoubleScriptFieldScript.LeafFactory, + Double> { + + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(randomDouble())))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(randomDouble())))); + }; + assertThat(execute(indexBuilder, "value(3.14)"), equalTo(List.of(3.14, 3.14))); + } + + public void testTwoConstants() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + }; + assertThat(execute(indexBuilder, "value(3.14); value(2.72)"), equalTo(List.of(3.14, 2.72, 3.14, 2.72))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); + }; + assertThat(execute(indexBuilder, "value(source['foo'] * 10.1)"), equalTo(List.of(10.1, 101.0))); + } + + public void testTwoSourceFields() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1, \"bar\": 2}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); + }; + assertThat( + execute(indexBuilder, "value(source['foo'] * 10.1); value(source['bar'] * 10.2)"), + equalTo(List.of(10.1, 20.4, 101.0, 204.0)) + ); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(1.1)))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(10.1)))); + }; + assertThat( + execute(indexBuilder, "value(doc['foo'].value * 9.9)", new NumberFieldType("foo", NumberType.DOUBLE)), + equalTo(List.of(10.89, 99.99)) + ); + } + + public void testTwoDocValuesValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument( + List.of( + new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(1.1)), + new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(2.2)) + ) + ); + iw.addDocument( + List.of( + new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(10.1)), + new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(20.1)) + ) + ); + }; + assertThat( + execute(indexBuilder, "for (double d : doc['foo']) {value(d * 9.9)}", new NumberFieldType("foo", NumberType.DOUBLE)), + equalTo(List.of(10.89, 21.78, 99.99, 198.99)) + ); + } + + @Override + protected ScriptContext scriptContext() { + return DoubleScriptFieldScript.CONTEXT; + } + + @Override + protected DoubleScriptFieldScript.LeafFactory newLeafFactory( + DoubleScriptFieldScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected DoubleScriptFieldScript newInstance( + DoubleScriptFieldScript.LeafFactory leafFactory, + LeafReaderContext context, + List result + ) throws IOException { + return leafFactory.newInstance(context, result::add); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index c5c40ffa0972b..2b8fc1bef2fbf 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -35,6 +35,8 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -44,7 +46,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public abstract class ScriptFieldScriptTestCase extends ESTestCase { +public abstract class ScriptFieldScriptTestCase extends ESTestCase { protected abstract ScriptContext scriptContext(); protected abstract LF newLeafFactory(F factory, Map params, SourceLookup source, DocLookup fieldData); @@ -67,7 +69,9 @@ protected final List execute(CheckedConsumer .build(indexSettings(), ft, null, new NoneCircuitBreakerService(), mapperService); DocLookup fieldData = new DocLookup(mapperService, fieldDataLookup); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { - F factory = scriptService.compile(new Script(script), scriptContext()); + F factory = AccessController.doPrivileged( + (PrivilegedAction) () -> scriptService.compile(new Script(script), scriptContext()) + ); try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { indexBuilder.accept(indexWriter); From 741a30fb7e16a68de9eab59dbf4f2ce1d3c7891f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 26 Jun 2020 11:00:50 -0400 Subject: [PATCH 05/82] Fixup test case after merge Painless extensions work differently now. --- .../xpack/runtimefields/ScriptFieldScriptTestCase.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 2b8fc1bef2fbf..a086953142662 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.painless.PainlessPlugin; +import org.elasticsearch.plugins.ExtensiblePlugin.ExtensionLoader; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptModule; @@ -57,7 +58,13 @@ protected final List execute(CheckedConsumer throws IOException { PainlessPlugin painlessPlugin = new PainlessPlugin(); - painlessPlugin.reloadSPI(Thread.currentThread().getContextClassLoader()); + painlessPlugin.loadExtensions(new ExtensionLoader() { + @Override + @SuppressWarnings("unchecked") // We only ever load painless extensions here so it is fairly safe. + public List loadExtensions(Class extensionPointType) { + return (List) List.of(new RuntimeFieldsPainlessExtension()); + } + }); ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); Map params = new HashMap<>(); SourceLookup source = new SourceLookup(); From 5011a736bccfcb9adb418608ba0c88c1794616f8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 26 Jun 2020 16:26:26 -0400 Subject: [PATCH 06/82] Get runtime fields tests passing Uses a temporary hack to allow us to use painless in unit tests. --- .../plugin-metadata/plugin-security.policy | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 x-pack/plugin/runtime-fields/src/main/plugin-metadata/plugin-security.policy diff --git a/x-pack/plugin/runtime-fields/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/runtime-fields/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..7683ec0535e17 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +/* TODO this exists so that we can run painless in unit tests which we + * only want to do because we don't yet have a FieldMapper implementation. + * Once we have that we can move to mock scripts in unit tests and painless + * in integration tests. */ + +grant { + // needed to generate runtime classes + permission java.lang.RuntimePermission "createClassLoader"; + + // needed to find the classloader to load whitelisted classes from + permission java.lang.RuntimePermission "getClassLoader"; +}; + From e833e8e09e331a512a1528f246610b834fa70bd8 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Thu, 2 Jul 2020 20:36:37 +0200 Subject: [PATCH 07/82] Scripted keyword field (#58939) This is the first draft implementation of scripted field of type keyword. Still lots of TODOs to address, this PR targets a feature branch. --- .../index/fielddata/SearchLookupAware.java | 28 +++ .../index/query/QueryShardContext.java | 9 +- .../AbstractScriptFieldScript.java | 24 +-- .../DoubleScriptFieldScript.java | 15 +- .../runtimefields/LongScriptFieldScript.java | 15 +- .../xpack/runtimefields/RuntimeFields.java | 37 +++- .../StringScriptFieldScript.java | 17 +- .../fielddata/ScriptBinaryDocValues.java | 41 ++++ .../fielddata/ScriptBinaryFieldData.java | 175 ++++++++++++++++++ .../mapper/RuntimeKeywordMappedFieldType.java | 85 +++++++++ .../mapper/ScriptFieldMapper.java | 138 ++++++++++++++ .../xpack/runtimefields/double_whitelist.txt | 6 +- .../xpack/runtimefields/long_whitelist.txt | 4 + .../xpack/runtimefields/string_whitelist.txt | 6 +- .../DoubleScriptFieldScriptTests.java | 8 +- .../LongScriptFieldScriptTests.java | 8 +- .../ScriptFieldScriptTestCase.java | 14 +- .../StringScriptFieldScriptTests.java | 8 +- 18 files changed, 562 insertions(+), 76 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java b/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java new file mode 100644 index 0000000000000..a7b33090ca6f9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java @@ -0,0 +1,28 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.elasticsearch.search.lookup.SearchLookup; + +//TODO this is a temporary interface only to avoid changing signature of MappedFieldType#fielddataBuilder +public interface SearchLookupAware { + + void setSearchLookup(SearchLookup searchLookup); +} diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index cc3a00bdd62a1..b550c6b5552b2 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.SearchLookupAware; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -208,7 +209,13 @@ public boolean allowExpensiveQueries() { @SuppressWarnings("unchecked") public > IFD getForField(MappedFieldType fieldType) { - return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName()); + IFD indexFieldData = (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName()); + //TODO this is a temporary hack to inject search lookup to the scripted fielddata + // implementations without changing MappedFieldType#fielddataBuilder signature, as that would cause daily merge conflicts + if (indexFieldData instanceof SearchLookupAware) { + ((SearchLookupAware) indexFieldData).setSearchLookup(lookup()); + } + return indexFieldData; } public void addNamedQuery(String name, Query query) { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java index b9d605c796530..ec49d38ef0a30 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -9,9 +9,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.script.AggregationScript; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.LeafDocLookup; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; import java.util.Map; @@ -21,23 +20,19 @@ */ public abstract class AbstractScriptFieldScript { private final Map params; - private final LeafReaderContext ctx; - private final SourceLookup source; - private final LeafDocLookup fieldData; + private final LeafSearchLookup leafSearchLookup; - public AbstractScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + public AbstractScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + this.leafSearchLookup = searchLookup.getLeafSearchLookup(ctx); + // TODO how do other scripts get stored fields exposed? Through asMap? I don't see any getters for them. this.params = params; - this.source = source; - this.fieldData = fieldData.getLeafDocLookup(ctx); - this.ctx = ctx; } /** * Set the document to run the script against. */ public final void setDocument(int docId) { - source.setSegmentAndDocument(ctx, docId); - fieldData.setDocument(docId); + this.leafSearchLookup.setDocument(docId); } /** @@ -51,16 +46,15 @@ public final Map getParams() { * Expose the {@code _source} to the script. */ public final Map getSource() { - return source; + return leafSearchLookup.source(); } /** * Expose field data to the script as {@code doc}. */ public final Map> getDoc() { - return fieldData; + return leafSearchLookup.doc(); } public abstract void execute(); - } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java index e86bfb20f71b6..8b66738b8bc74 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -11,8 +11,7 @@ import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; @@ -29,7 +28,7 @@ static List whitelist() { public static final String[] PARAMETERS = {}; public interface Factory extends ScriptFactory { - LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + LeafFactory newFactory(Map params, SearchLookup searchLookup); } public interface LeafFactory { @@ -38,14 +37,8 @@ public interface LeafFactory { private final DoubleConsumer sync; - public DoubleScriptFieldScript( - Map params, - SourceLookup source, - DocLookup fieldData, - LeafReaderContext ctx, - DoubleConsumer sync - ) { - super(params, source, fieldData, ctx); + public DoubleScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx, DoubleConsumer sync) { + super(params, searchLookup, ctx); this.sync = sync; } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index e99fa4a944ba3..9850cab424642 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -11,8 +11,7 @@ import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; @@ -29,7 +28,7 @@ static List whitelist() { public static final String[] PARAMETERS = {}; public interface Factory extends ScriptFactory { - LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + LeafFactory newFactory(Map params, SearchLookup searchLookup); } public interface LeafFactory { @@ -38,14 +37,8 @@ public interface LeafFactory { private final LongConsumer sync; - public LongScriptFieldScript( - Map params, - SourceLookup source, - DocLookup fieldData, - LeafReaderContext ctx, - LongConsumer sync - ) { - super(params, source, fieldData, ctx); + public LongScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx, LongConsumer sync) { + super(params, searchLookup, ctx); this.sync = sync; } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 6b97b0dcb795c..f41c7efd44e69 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -6,25 +6,60 @@ package org.elasticsearch.xpack.runtimefields; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.runtimefields.mapper.ScriptFieldMapper; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { + private final ScriptFieldMapper.TypeParser scriptTypeParser = new ScriptFieldMapper.TypeParser(); + @Override public Map getMappers() { - return Collections.emptyMap(); + return Collections.singletonMap(ScriptFieldMapper.CONTENT_TYPE, scriptTypeParser); } @Override public List> getContexts() { return List.of(DoubleScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT, StringScriptFieldScript.CONTEXT); } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + // looks like createComponents gets called after getMappers + this.scriptTypeParser.setScriptService(scriptService); + return Collections.emptyList(); + } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 2556381981236..391be84b8d964 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -11,8 +11,7 @@ import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; @@ -20,7 +19,7 @@ import java.util.function.Consumer; public abstract class StringScriptFieldScript extends AbstractScriptFieldScript { - static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); + public static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); static List whitelist() { return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt")); @@ -29,7 +28,7 @@ static List whitelist() { public static final String[] PARAMETERS = {}; public interface Factory extends ScriptFactory { - LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + LeafFactory newFactory(Map params, SearchLookup searchLookup); } public interface LeafFactory { @@ -38,14 +37,8 @@ public interface LeafFactory { private final Consumer sync; - public StringScriptFieldScript( - Map params, - SourceLookup source, - DocLookup fieldData, - LeafReaderContext ctx, - Consumer sync - ) { - super(params, source, fieldData, ctx); + public StringScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx, Consumer sync) { + super(params, searchLookup, ctx); this.sync = sync; } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java new file mode 100644 index 0000000000000..ae0edeae46b4e --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.SortingBinaryDocValues; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +public final class ScriptBinaryDocValues extends SortingBinaryDocValues { + + private final StringScriptFieldScript script; + private final ScriptBinaryFieldData.ScriptBinaryResult scriptBinaryResult; + + ScriptBinaryDocValues(StringScriptFieldScript script, ScriptBinaryFieldData.ScriptBinaryResult scriptBinaryResult) { + this.script = script; + this.scriptBinaryResult = scriptBinaryResult; + } + + @Override + public boolean advanceExact(int doc) { + script.setDocument(doc); + script.execute(); + + count = scriptBinaryResult.getResult().size(); + if (count == 0) { + grow(); + return false; + } + + int i = 0; + for (String value : scriptBinaryResult.getResult()) { + grow(); + values[i++].copyChars(value); + } + sort(); + return true; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java new file mode 100644 index 0000000000000..42ca35731274b --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.AbstractIndexComponent; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SearchLookupAware; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public final class ScriptBinaryFieldData extends AbstractIndexComponent + implements + IndexFieldData, + SearchLookupAware { + + public static class Builder implements IndexFieldData.Builder { + + private final StringScriptFieldScript.Factory scriptFactory; + + public Builder(StringScriptFieldScript.Factory scriptFactory) { + this.scriptFactory = scriptFactory; + } + + @Override + public IndexFieldData build( + IndexSettings indexSettings, + MappedFieldType fieldType, + IndexFieldDataCache cache, + CircuitBreakerService breakerService, + MapperService mapperService + ) { + return new ScriptBinaryFieldData(indexSettings, fieldType.name(), scriptFactory); + } + } + + private final String fieldName; + private final StringScriptFieldScript.Factory scriptFactory; + private final SetOnce leafFactory = new SetOnce<>(); + + private ScriptBinaryFieldData(IndexSettings indexSettings, String fieldName, StringScriptFieldScript.Factory scriptFactory) { + super(indexSettings); + this.fieldName = fieldName; + this.scriptFactory = scriptFactory; + } + + public void setSearchLookup(SearchLookup searchLookup) { + // TODO wire the params from the mappings definition, we don't parse them yet + this.leafFactory.set(scriptFactory.newFactory(Collections.emptyMap(), searchLookup)); + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } + + @Override + public ScriptBinaryLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + if (e instanceof ElasticsearchException) { + throw (ElasticsearchException) e; + } else { + throw new ElasticsearchException(e); + } + } + } + + @Override + public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + ScriptBinaryResult scriptBinaryResult = new ScriptBinaryResult(); + return new ScriptBinaryLeafFieldData( + new ScriptBinaryDocValues(leafFactory.get().newInstance(context, scriptBinaryResult::accept), scriptBinaryResult) + ); + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); + return new SortField(getFieldName(), source, reverse); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("only supported on numeric fields"); + } + + @Override + public void clear() { + + } + + static class ScriptBinaryLeafFieldData implements LeafFieldData { + private final ScriptBinaryDocValues scriptBinaryDocValues; + + ScriptBinaryLeafFieldData(ScriptBinaryDocValues scriptBinaryDocValues) { + this.scriptBinaryDocValues = scriptBinaryDocValues; + } + + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues.Strings(getBytesValues()); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return scriptBinaryDocValues; + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() { + + } + } + + static class ScriptBinaryResult { + private final List result = new ArrayList<>(); + + void accept(String value) { + this.result.add(value); + } + + List getResult() { + return result; + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java new file mode 100644 index 0000000000000..5aef8e857408a --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptBinaryFieldData; + +import java.io.IOException; +import java.util.Map; + +public final class RuntimeKeywordMappedFieldType extends MappedFieldType { + + private final Script script; + private final StringScriptFieldScript.Factory scriptFactory; + + RuntimeKeywordMappedFieldType(String name, Script script, StringScriptFieldScript.Factory scriptFactory, Map meta) { + super(name, false, false, TextSearchInfo.NONE, meta); + this.script = script; + this.scriptFactory = scriptFactory; + } + + RuntimeKeywordMappedFieldType(RuntimeKeywordMappedFieldType ref) { + super(ref); + this.script = ref.script; + this.scriptFactory = ref.scriptFactory; + } + + @Override + public MappedFieldType clone() { + return new RuntimeKeywordMappedFieldType(this); + } + + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + // keywords are internally stored as utf8 bytes + BytesRef binaryValue = (BytesRef) value; + return binaryValue.utf8ToString(); + } + + @Override + public String typeName() { + // TODO not sure what we should return here: the runtime type or the field type? + // why is the same string returned from three different methods? + return ScriptFieldMapper.CONTENT_TYPE; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + // TODO once we get SearchLookup as an argument, we can already call scriptFactory.newFactory here and pass through the result + return new ScriptBinaryFieldData.Builder(scriptFactory); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return null; + } + + void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + builder.field("runtime_type", "keyword"); + builder.field("script", script.getIdOrCode()); // TODO For some reason this doesn't allow us to do the full xcontent of the script. + } + + // TODO do we need to override equals/hashcode? +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java new file mode 100644 index 0000000000000..c6fcd43635865 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.document.FieldType; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public final class ScriptFieldMapper extends FieldMapper { + + public static final String CONTENT_TYPE = "script"; + + private static final FieldType FIELD_TYPE = new FieldType(); + + ScriptFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { + super(simpleName, FIELD_TYPE, mappedFieldType, multiFields, copyTo); + } + + @Override + protected void parseCreateField(ParseContext context) { + // there is no field! + } + + @Override + protected void mergeOptions(FieldMapper other, List conflicts) { + // TODO implement this + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + RuntimeKeywordMappedFieldType fieldType = (RuntimeKeywordMappedFieldType) fieldType(); + fieldType.doXContentBody(builder, includeDefaults, params); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + public static class Builder extends FieldMapper.Builder { + + private final ScriptService scriptService; + + private String runtimeType; + private Script script; + + protected Builder(String name, ScriptService scriptService) { + super(name, FIELD_TYPE); + this.scriptService = scriptService; + } + + public void runtimeType(String runtimeType) { + this.runtimeType = runtimeType; + } + + public void script(Script script) { + this.script = script; + } + + @Override + public ScriptFieldMapper build(BuilderContext context) { + if (runtimeType == null) { + throw new IllegalArgumentException("runtime_type must be specified"); + } + if (script == null) { + throw new IllegalArgumentException("script must be specified"); + } + + MappedFieldType mappedFieldType; + if (runtimeType.equals("keyword")) { + StringScriptFieldScript.Factory factory = scriptService.compile(script, StringScriptFieldScript.CONTEXT); + mappedFieldType = new RuntimeKeywordMappedFieldType(buildFullName(context), script, factory, meta); + } else { + throw new IllegalArgumentException("runtime_type [" + runtimeType + "] not supported"); + } + // TODO copy to and multi_fields... not sure what needs to be done. + return new ScriptFieldMapper(name, mappedFieldType, multiFieldsBuilder.build(this, context), copyTo); + } + } + + public static class TypeParser implements Mapper.TypeParser { + + private final SetOnce scriptService = new SetOnce<>(); + + public void setScriptService(ScriptService scriptService) { + this.scriptService.set(scriptService); + } + + @Override + public ScriptFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) + throws MapperParsingException { + Builder builder = new Builder(name, scriptService.get()); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = entry.getKey(); + Object propNode = entry.getValue(); + if (propName.equals("runtime_type")) { + if (propNode == null) { + throw new MapperParsingException("Property [runtime_type] cannot be null."); + } + builder.runtimeType(XContentMapValues.nodeStringValue(propNode, name + ".runtime_type")); + iterator.remove(); + } else if (propName.equals("script")) { + if (propNode == null) { + throw new MapperParsingException("Property [script] cannot be null."); + } + // TODO this should become an object and support the usual script syntax, including lang and params + builder.script(new Script(XContentMapValues.nodeStringValue(propNode, name + ".script"))); + iterator.remove(); + } + } + // TODO these get passed in sometimes and we don't need them + node.remove("doc_values"); + node.remove("index"); + return builder; + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt index 55a177f21579c..ce8963b6f7cbf 100644 --- a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the Elastic License. # -# The whitelist for long-valued runtime fields +# The whitelist for double-valued runtime fields class org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript @no_import { } @@ -12,3 +12,7 @@ class org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript @no_import { static_import { void value(org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript, double) bound_to org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript$Value } + +# This import is required to make painless happy and it isn't 100% clear why +class org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript$Factory @no_import { +} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt index 630c679bc21cc..f5f3d0245f949 100644 --- a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt @@ -12,3 +12,7 @@ class org.elasticsearch.xpack.runtimefields.LongScriptFieldScript @no_import { static_import { void value(org.elasticsearch.xpack.runtimefields.LongScriptFieldScript, long) bound_to org.elasticsearch.xpack.runtimefields.LongScriptFieldScript$Value } + +# This import is required to make painless happy and it isn't 100% clear why +class org.elasticsearch.xpack.runtimefields.LongScriptFieldScript$Factory @no_import { +} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt index e0c4e367850f3..5570891fb4b19 100644 --- a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the Elastic License. # -# The whitelist for long-valued runtime fields +# The whitelist for string-valued runtime fields class org.elasticsearch.xpack.runtimefields.StringScriptFieldScript @no_import { } @@ -12,3 +12,7 @@ class org.elasticsearch.xpack.runtimefields.StringScriptFieldScript @no_import { static_import { void value(org.elasticsearch.xpack.runtimefields.StringScriptFieldScript, String) bound_to org.elasticsearch.xpack.runtimefields.StringScriptFieldScript$Value } + +# This import is required to make painless happy and it isn't 100% clear why +class org.elasticsearch.xpack.runtimefields.StringScriptFieldScript$Factory @no_import { +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 37c6c5d35ce41..23081d1ff3427 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -16,8 +16,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; @@ -107,10 +106,9 @@ protected ScriptContext scriptContext() { protected DoubleScriptFieldScript.LeafFactory newLeafFactory( DoubleScriptFieldScript.Factory factory, Map params, - SourceLookup source, - DocLookup fieldData + SearchLookup searchLookup ) { - return factory.newFactory(params, source, fieldData); + return factory.newFactory(params, searchLookup); } @Override diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 05d48808b5620..ef0c5cd304d0e 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -15,8 +15,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; @@ -93,10 +92,9 @@ protected ScriptContext scriptContext() { protected LongScriptFieldScript.LeafFactory newLeafFactory( LongScriptFieldScript.Factory factory, Map params, - SourceLookup source, - DocLookup fieldData + SearchLookup searchLookup ) { - return factory.newFactory(params, source, fieldData); + return factory.newFactory(params, searchLookup); } @Override diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index a086953142662..3b7a819d04b40 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -31,8 +31,7 @@ import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -50,7 +49,7 @@ public abstract class ScriptFieldScriptTestCase extends ESTestCase { protected abstract ScriptContext scriptContext(); - protected abstract LF newLeafFactory(F factory, Map params, SourceLookup source, DocLookup fieldData); + protected abstract LF newLeafFactory(F factory, Map params, SearchLookup searchLookup); protected abstract S newInstance(LF leafFactory, LeafReaderContext context, List results) throws IOException; @@ -67,14 +66,13 @@ public List loadExtensions(Class extensionPointType) { }); ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); Map params = new HashMap<>(); - SourceLookup source = new SourceLookup(); MapperService mapperService = mock(MapperService.class); for (MappedFieldType type : types) { when(mapperService.fieldType(type.name())).thenReturn(type); } Function> fieldDataLookup = ft -> ft.fielddataBuilder("test") .build(indexSettings(), ft, null, new NoneCircuitBreakerService(), mapperService); - DocLookup fieldData = new DocLookup(mapperService, fieldDataLookup); + SearchLookup searchLookup = new SearchLookup(mapperService, fieldDataLookup); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { F factory = AccessController.doPrivileged( (PrivilegedAction) () -> scriptService.compile(new Script(script), scriptContext()) @@ -84,7 +82,7 @@ public List loadExtensions(Class extensionPointType) { indexBuilder.accept(indexWriter); try (DirectoryReader reader = indexWriter.getReader()) { IndexSearcher searcher = newSearcher(reader); - LF leafFactory = newLeafFactory(factory, params, source, fieldData); + LF leafFactory = newLeafFactory(factory, params, searchLookup); List result = new ArrayList<>(); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override @@ -97,10 +95,10 @@ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOExcept S compiled = newInstance(leafFactory, context, result); return new LeafCollector() { @Override - public void setScorer(Scorable scorer) throws IOException {} + public void setScorer(Scorable scorer) {} @Override - public void collect(int doc) throws IOException { + public void collect(int doc) { compiled.setDocument(doc); compiled.execute(); } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 658001f743281..5ecedd1762647 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -14,8 +14,7 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; @@ -99,10 +98,9 @@ protected ScriptContext scriptContext() { protected StringScriptFieldScript.LeafFactory newLeafFactory( StringScriptFieldScript.Factory factory, Map params, - SourceLookup source, - DocLookup fieldData + SearchLookup searchLookup ) { - return factory.newFactory(params, source, fieldData); + return factory.newFactory(params, searchLookup); } @Override From 949d1de0be1eef3062c617673bc5437f04934012 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 10 Jul 2020 15:53:47 +0200 Subject: [PATCH 08/82] adapt to upstream changes --- .../mapper/RuntimeKeywordMappedFieldType.java | 13 ------------- .../runtimefields/mapper/ScriptFieldMapper.java | 3 +-- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java index 5aef8e857408a..764d9f0cb61e6 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java @@ -32,17 +32,6 @@ public final class RuntimeKeywordMappedFieldType extends MappedFieldType { this.scriptFactory = scriptFactory; } - RuntimeKeywordMappedFieldType(RuntimeKeywordMappedFieldType ref) { - super(ref); - this.script = ref.script; - this.scriptFactory = ref.scriptFactory; - } - - @Override - public MappedFieldType clone() { - return new RuntimeKeywordMappedFieldType(this); - } - @Override public Object valueForDisplay(Object value) { if (value == null) { @@ -80,6 +69,4 @@ void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params par builder.field("runtime_type", "keyword"); builder.field("script", script.getIdOrCode()); // TODO For some reason this doesn't allow us to do the full xcontent of the script. } - - // TODO do we need to override equals/hashcode? } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java index c6fcd43635865..3d0135987bb03 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java @@ -10,7 +10,6 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -57,7 +56,7 @@ protected String contentType() { return CONTENT_TYPE; } - public static class Builder extends FieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final ScriptService scriptService; From 265964863fbb163e72d8f181887da227dbbd5d4e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 13 Jul 2020 17:23:03 -0400 Subject: [PATCH 09/82] Add term query for keyword script fields (#59372) This adds what I think is just about the simplest possible `term` query implementation for `keyword` script fields and wires it into the field mapper that we build for them. --- .../StringScriptFieldScript.java | 24 +++-- .../fielddata/ScriptBinaryDocValues.java | 20 ++-- .../fielddata/ScriptBinaryFieldData.java | 5 +- .../mapper/RuntimeKeywordMappedFieldType.java | 11 ++- .../query/StringScriptFieldTermQuery.java | 97 +++++++++++++++++++ .../DoubleScriptFieldScriptTests.java | 19 ++-- .../LongScriptFieldScriptTests.java | 15 ++- .../ScriptFieldScriptTestCase.java | 12 +-- .../StringScriptFieldScriptTests.java | 11 +-- .../RuntimeKeywordMappedFieldTypeTests.java | 72 ++++++++++++++ .../test/runtime_fields/10_keyword.yml | 53 ++++++++++ 11 files changed, 291 insertions(+), 48 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldTermQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 391be84b8d964..09321de9a225d 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -14,9 +14,9 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.function.Consumer; public abstract class StringScriptFieldScript extends AbstractScriptFieldScript { public static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); @@ -32,14 +32,26 @@ public interface Factory extends ScriptFactory { } public interface LeafFactory { - StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; + StringScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; } - private final Consumer sync; + private final List results = new ArrayList<>(); - public StringScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx, Consumer sync) { + public StringScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { super(params, searchLookup, ctx); - this.sync = sync; + } + + /** + * Execute the script for the provided {@code docId}. + *

+ * @return a mutable {@link List} that contains the results of the script + * and will be modified the next time you call {@linkplain #resultsForDoc}. + */ + public final List resultsForDoc(int docId) { + results.clear(); + setDocument(docId); + execute(); + return results; } public static class Value { @@ -50,7 +62,7 @@ public Value(StringScriptFieldScript script) { } public void value(String v) { - script.sync.accept(v); + script.results.add(v); } } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java index ae0edeae46b4e..f43ed950b4dd7 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java @@ -9,30 +9,26 @@ import org.elasticsearch.index.fielddata.SortingBinaryDocValues; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; -public final class ScriptBinaryDocValues extends SortingBinaryDocValues { +import java.util.List; +public final class ScriptBinaryDocValues extends SortingBinaryDocValues { private final StringScriptFieldScript script; - private final ScriptBinaryFieldData.ScriptBinaryResult scriptBinaryResult; - ScriptBinaryDocValues(StringScriptFieldScript script, ScriptBinaryFieldData.ScriptBinaryResult scriptBinaryResult) { + ScriptBinaryDocValues(StringScriptFieldScript script) { this.script = script; - this.scriptBinaryResult = scriptBinaryResult; } @Override - public boolean advanceExact(int doc) { - script.setDocument(doc); - script.execute(); - - count = scriptBinaryResult.getResult().size(); + public boolean advanceExact(int docId) { + List results = script.resultsForDoc(docId); + count = results.size(); if (count == 0) { - grow(); return false; } + grow(); int i = 0; - for (String value : scriptBinaryResult.getResult()) { - grow(); + for (String value : results) { values[i++].copyChars(value); } sort(); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java index 42ca35731274b..2a21efc52a387 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java @@ -102,10 +102,7 @@ public ScriptBinaryLeafFieldData load(LeafReaderContext context) { @Override public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws IOException { - ScriptBinaryResult scriptBinaryResult = new ScriptBinaryResult(); - return new ScriptBinaryLeafFieldData( - new ScriptBinaryDocValues(leafFactory.get().newInstance(context, scriptBinaryResult::accept), scriptBinaryResult) - ); + return new ScriptBinaryLeafFieldData(new ScriptBinaryDocValues(leafFactory.get().newInstance(context))); } @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java index 764d9f0cb61e6..c8680da19a8c5 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java @@ -8,8 +8,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -17,9 +18,11 @@ import org.elasticsearch.script.Script; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; import org.elasticsearch.xpack.runtimefields.fielddata.ScriptBinaryFieldData; +import org.elasticsearch.xpack.runtimefields.query.StringScriptFieldTermQuery; import java.io.IOException; import java.util.Map; +import java.util.Objects; public final class RuntimeKeywordMappedFieldType extends MappedFieldType { @@ -57,7 +60,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public Query termQuery(Object value, QueryShardContext context) { - return null; + return new StringScriptFieldTermQuery( + scriptFactory.newFactory(script.getParams(), context.lookup()), + name(), + BytesRefs.toString(Objects.requireNonNull(value)) + ); } @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldTermQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldTermQuery.java new file mode 100644 index 0000000000000..4d1f319884158 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldTermQuery.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +import java.io.IOException; +import java.util.Objects; + +public class StringScriptFieldTermQuery extends Query { + private final StringScriptFieldScript.LeafFactory leafFactory; + private final String fieldName; + private final String term; + + public StringScriptFieldTermQuery(StringScriptFieldScript.LeafFactory leafFactory, String fieldName, String term) { + this.leafFactory = leafFactory; + this.fieldName = fieldName; + this.term = term; + } + + @Override + public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } + + @Override + public Scorer scorer(LeafReaderContext ctx) throws IOException { + StringScriptFieldScript script = leafFactory.newInstance(ctx); + DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override + public boolean matches() throws IOException { + for (String result : script.resultsForDoc(approximation().docID())) { + if (term.equals(result)) { + return true; + } + } + return false; + } + + @Override + public float matchCost() { + // TODO we don't have a good way of estimating the complexity of the script so we just go with 9000 + return 9000f; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + }; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.consumeTerms(this, new Term(fieldName, term)); + } + + @Override + public final String toString(String field) { + if (fieldName.contentEquals(field)) { + return term; + } + return fieldName + ":" + term; + } + + @Override + public int hashCode() { + return Objects.hash(fieldName, term); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + StringScriptFieldTermQuery other = (StringScriptFieldTermQuery) obj; + return fieldName.equals(other.fieldName) && term.equals(other.term); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 23081d1ff3427..88d1232531e56 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -19,13 +19,14 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.IntFunction; import static org.hamcrest.Matchers.equalTo; public class DoubleScriptFieldScriptTests extends ScriptFieldScriptTestCase< - DoubleScriptFieldScript, DoubleScriptFieldScript.Factory, DoubleScriptFieldScript.LeafFactory, Double> { @@ -112,11 +113,15 @@ protected DoubleScriptFieldScript.LeafFactory newLeafFactory( } @Override - protected DoubleScriptFieldScript newInstance( - DoubleScriptFieldScript.LeafFactory leafFactory, - LeafReaderContext context, - List result - ) throws IOException { - return leafFactory.newInstance(context, result::add); + protected IntFunction> newInstance(DoubleScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) + throws IOException { + List results = new ArrayList<>(); + DoubleScriptFieldScript script = leafFactory.newInstance(context, results::add); + return docId -> { + results.clear(); + script.setDocument(docId); + script.execute(); + return results; + }; } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index ef0c5cd304d0e..d8d8da6d715f4 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -18,13 +18,14 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.IntFunction; import static org.hamcrest.Matchers.equalTo; public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< - LongScriptFieldScript, LongScriptFieldScript.Factory, LongScriptFieldScript.LeafFactory, Long> { @@ -98,9 +99,15 @@ protected LongScriptFieldScript.LeafFactory newLeafFactory( } @Override - protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context, List result) + protected IntFunction> newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) throws IOException { - - return leafFactory.newInstance(context, result::add); + List results = new ArrayList<>(); + LongScriptFieldScript script = leafFactory.newInstance(context, results::add); + return docId -> { + results.clear(); + script.setDocument(docId); + script.execute(); + return results; + }; } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 3b7a819d04b40..d2580a8ad5b8e 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -42,16 +42,17 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.function.IntFunction; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public abstract class ScriptFieldScriptTestCase extends ESTestCase { +public abstract class ScriptFieldScriptTestCase extends ESTestCase { protected abstract ScriptContext scriptContext(); protected abstract LF newLeafFactory(F factory, Map params, SearchLookup searchLookup); - protected abstract S newInstance(LF leafFactory, LeafReaderContext context, List results) throws IOException; + protected abstract IntFunction> newInstance(LF leafFactory, LeafReaderContext context) throws IOException; protected final List execute(CheckedConsumer indexBuilder, String script, MappedFieldType... types) throws IOException { @@ -92,15 +93,14 @@ public ScoreMode scoreMode() { @Override public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - S compiled = newInstance(leafFactory, context, result); + IntFunction> compiled = newInstance(leafFactory, context); return new LeafCollector() { @Override public void setScorer(Scorable scorer) {} @Override - public void collect(int doc) { - compiled.setDocument(doc); - compiled.execute(); + public void collect(int docId) { + result.addAll(compiled.apply(docId)); } }; } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 5ecedd1762647..15d6d225b14e5 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -19,11 +19,11 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.IntFunction; import static org.hamcrest.Matchers.equalTo; public class StringScriptFieldScriptTests extends ScriptFieldScriptTestCase< - StringScriptFieldScript, StringScriptFieldScript.Factory, StringScriptFieldScript.LeafFactory, String> { @@ -104,11 +104,8 @@ protected StringScriptFieldScript.LeafFactory newLeafFactory( } @Override - protected StringScriptFieldScript newInstance( - StringScriptFieldScript.LeafFactory leafFactory, - LeafReaderContext context, - List result - ) throws IOException { - return leafFactory.newInstance(context, result::add); + protected IntFunction> newInstance(StringScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) + throws IOException { + return leafFactory.newInstance(context)::resultsForDoc; } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java new file mode 100644 index 0000000000000..ba09555da47d5 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldTypeTests.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.painless.PainlessPlugin; +import org.elasticsearch.plugins.ExtensiblePlugin.ExtensionLoader; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.runtimefields.RuntimeFields; +import org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RuntimeKeywordMappedFieldTypeTests extends ESTestCase { + public void testTermQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 2}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat(searcher.count(build("value(source.foo.toString())").termQuery("1", mockContext())), equalTo(1)); + } + } + } + + private RuntimeKeywordMappedFieldType build(String code) throws IOException { + Script script = new Script(code); + PainlessPlugin painlessPlugin = new PainlessPlugin(); + painlessPlugin.loadExtensions(new ExtensionLoader() { + @Override + @SuppressWarnings("unchecked") // We only ever load painless extensions here so it is fairly safe. + public List loadExtensions(Class extensionPointType) { + return (List) List.of(new RuntimeFieldsPainlessExtension()); + } + }); + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); + try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { + StringScriptFieldScript.Factory factory = scriptService.compile(script, StringScriptFieldScript.CONTEXT); + return new RuntimeKeywordMappedFieldType("test", script, factory, emptyMap()); + } + } + + private QueryShardContext mockContext() { + MapperService mapperService = mock(MapperService.class); + QueryShardContext context = mock(QueryShardContext.class); + when(context.lookup()).thenReturn(new SearchLookup(mapperService, mft -> null)); + return context; + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml new file mode 100644 index 0000000000000..5d30364789b5b --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -0,0 +1,53 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: float + node: + type: keyword + day_of_week: + type: script + runtime_type: keyword + script: value(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)) + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} From 4eb18b28f0677dcd4a47415d9a05007baecf6129 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 14 Jul 2020 16:57:31 +0200 Subject: [PATCH 10/82] Make ScriptFieldMapper a parameterized mapper (#59391) Relates to #59332 --- .../xpack/runtimefields/RuntimeFields.java | 8 +- .../StringScriptFieldScript.java | 2 +- .../mapper/ScriptFieldMapper.java | 140 ++++++++------- .../mapper/ScriptFieldMapperTests.java | 160 ++++++++++++++++++ 4 files changed, 247 insertions(+), 63 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapperTests.java diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index f41c7efd44e69..167960c33dc4d 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -32,11 +32,11 @@ public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { - private final ScriptFieldMapper.TypeParser scriptTypeParser = new ScriptFieldMapper.TypeParser(); + private final ScriptFieldMapper.TypeParser typeParser = new ScriptFieldMapper.TypeParser(); @Override public Map getMappers() { - return Collections.singletonMap(ScriptFieldMapper.CONTENT_TYPE, scriptTypeParser); + return Collections.singletonMap(ScriptFieldMapper.CONTENT_TYPE, typeParser); } @Override @@ -58,8 +58,8 @@ public Collection createComponents( IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier ) { - // looks like createComponents gets called after getMappers - this.scriptTypeParser.setScriptService(scriptService); + // TODO getMappers gets called before createComponents. We should wire the script service differently + typeParser.setScriptService(scriptService); return Collections.emptyList(); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 09321de9a225d..69a196446aaf0 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -35,7 +35,7 @@ public interface LeafFactory { StringScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; } - private final List results = new ArrayList<>(); + protected final List results = new ArrayList<>(); public StringScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { super(params, searchLookup, ctx); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java index 3d0135987bb03..9826d1bbd07dc 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptFieldMapper.java @@ -6,42 +6,54 @@ package org.elasticsearch.xpack.runtimefields.mapper; -import org.apache.lucene.document.FieldType; import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; import java.io.IOException; -import java.util.Iterator; import java.util.List; import java.util.Map; -public final class ScriptFieldMapper extends FieldMapper { +public final class ScriptFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "script"; - private static final FieldType FIELD_TYPE = new FieldType(); - - ScriptFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, FIELD_TYPE, mappedFieldType, multiFields, copyTo); + private final String runtimeType; + private final Script script; + private final ScriptService scriptService; + + protected ScriptFieldMapper( + String simpleName, + MappedFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + String runtimeType, + Script script, + ScriptService scriptService + ) { + super(simpleName, mappedFieldType, multiFields, copyTo); + this.runtimeType = runtimeType; + this.script = script; + this.scriptService = scriptService; } @Override - protected void parseCreateField(ParseContext context) { - // there is no field! + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new ScriptFieldMapper.Builder(simpleName(), scriptService).init(this); } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - // TODO implement this + protected void parseCreateField(ParseContext context) { + // there is no lucene field } @Override @@ -56,44 +68,76 @@ protected String contentType() { return CONTENT_TYPE; } - public static class Builder extends FieldMapper.Builder { + public static class Builder extends ParametrizedFieldMapper.Builder { - private final ScriptService scriptService; + private static ScriptFieldMapper toType(FieldMapper in) { + return (ScriptFieldMapper) in; + } + + private final Parameter> meta = Parameter.metaParam(); + private final Parameter runtimeType = Parameter.stringParam( + "runtime_type", + true, + mapper -> toType(mapper).runtimeType, + null + ).setValidator(runtimeType -> { + if (runtimeType == null) { + throw new IllegalArgumentException("runtime_type must be specified for script field [" + name + "]"); + } + }); + // TODO script and runtime_type can be updated: what happens to the currently running queries when they get updated? + // do all the shards get a consistent view? + private final Parameter