From 7553f7912fd16bbae89fcfc7378094a79541b19e Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 18 Jan 2022 11:56:44 -0800 Subject: [PATCH 1/4] Make wildcard accessible from the scripting fields API --- .../plain/StringBinaryDVLeafFieldData.java | 13 ++++-- .../plain/StringBinaryIndexFieldData.java | 12 +++++- x-pack/plugin/wildcard/build.gradle | 4 +- .../wildcard/WildcardDocValuesField.java | 18 ++++++++ .../wildcard/WildcardPainlessExtension.java | 37 ++++++++++++++++ .../wildcard/mapper/WildcardFieldMapper.java | 7 +++- ...asticsearch.painless.spi.PainlessExtension | 1 + .../org.elasticsearch.xpack.wildcard.txt | 12 ++++++ .../WildcardClientYamlTestSuiteIT.java | 27 ++++++++++++ .../rest-api-spec/test/10_script_values.yml | 42 +++++++++++++++++++ 10 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/WildcardDocValuesField.java create mode 100644 x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/WildcardPainlessExtension.java create mode 100644 x-pack/plugin/wildcard/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/wildcard/src/main/resources/org/elasticsearch/xpack/wildcard/org.elasticsearch.xpack.wildcard.txt create mode 100644 x-pack/plugin/wildcard/src/yamlRestTest/java/org/elasticsearch/xpack/wildcard/WildcardClientYamlTestSuiteIT.java create mode 100644 x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java index bbd5dd486e034..9039ca8989069 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java @@ -9,17 +9,22 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.BinaryDocValues; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.script.field.DocValuesField; +import org.elasticsearch.script.field.ToScriptField; final class StringBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData { - StringBinaryDVLeafFieldData(BinaryDocValues values) { + + protected final ToScriptField toScriptField; + + StringBinaryDVLeafFieldData(BinaryDocValues values, ToScriptField toScriptField) { super(values); + + this.toScriptField = toScriptField; } @Override public DocValuesField getScriptField(String name) { - return new DelegateDocValuesField(new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(getBytesValues())), name); + return toScriptField.getScriptField(getBytesValues(), name); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java index 995cf6d47cba4..e52a97aae6824 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java @@ -14,7 +14,9 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.script.field.ToScriptField; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -27,10 +29,16 @@ public class StringBinaryIndexFieldData implements IndexFieldData toScriptField; - public StringBinaryIndexFieldData(String fieldName, ValuesSourceType valuesSourceType) { + public StringBinaryIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ToScriptField toScriptField + ) { this.fieldName = fieldName; this.valuesSourceType = valuesSourceType; + this.toScriptField = toScriptField; } @Override @@ -52,7 +60,7 @@ public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested @Override public StringBinaryDVLeafFieldData load(LeafReaderContext context) { try { - return new StringBinaryDVLeafFieldData(DocValues.getBinary(context.reader(), fieldName)); + return new StringBinaryDVLeafFieldData(DocValues.getBinary(context.reader(), fieldName), toScriptField); } catch (IOException e) { throw new IllegalStateException("Cannot load doc values", e); } diff --git a/x-pack/plugin/wildcard/build.gradle b/x-pack/plugin/wildcard/build.gradle index 39620245c024e..82f2d33ac0041 100644 --- a/x-pack/plugin/wildcard/build.gradle +++ b/x-pack/plugin/wildcard/build.gradle @@ -1,16 +1,18 @@ import org.elasticsearch.gradle.internal.info.BuildParams apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-yaml-rest-test' esplugin { name 'wildcard' description 'A plugin for a keyword field type with efficient wildcard search' classname 'org.elasticsearch.xpack.wildcard.Wildcard' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['x-pack-core', 'lang-painless'] } archivesBaseName = 'x-pack-wildcard' dependencies { + compileOnly project(':modules:lang-painless:spi') compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/WildcardDocValuesField.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/WildcardDocValuesField.java new file mode 100644 index 0000000000000..60124aa85065e --- /dev/null +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/WildcardDocValuesField.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.wildcard; + +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.script.field.AbstractKeywordDocValuesField; + +public class WildcardDocValuesField extends AbstractKeywordDocValuesField { + + public WildcardDocValuesField(SortedBinaryDocValues input, String name) { + super(input, name); + } +} diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/WildcardPainlessExtension.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/WildcardPainlessExtension.java new file mode 100644 index 0000000000000..3f328750d5141 --- /dev/null +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/WildcardPainlessExtension.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.wildcard; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.script.ScriptModule.CORE_CONTEXTS; + +public class WildcardPainlessExtension implements PainlessExtension { + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles( + WildcardPainlessExtension.class, + "org.elasticsearch.xpack.wildcard.txt" + ); + + @Override + public Map, List> getContextWhitelists() { + List whitelist = singletonList(WHITELIST); + Map, List> contextWhitelists = new HashMap<>(CORE_CONTEXTS.size()); + for (ScriptContext scriptContext : CORE_CONTEXTS.values()) { + contextWhitelists.put(scriptContext, whitelist); + } + return contextWhitelists; + } +} diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index d6af156282da9..de17e2108c8a8 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -67,6 +67,7 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.wildcard.WildcardDocValuesField; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -829,7 +830,11 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); - return (cache, breakerService) -> new StringBinaryIndexFieldData(name(), CoreValuesSourceType.KEYWORD); + return (cache, breakerService) -> new StringBinaryIndexFieldData( + name(), + CoreValuesSourceType.KEYWORD, + WildcardDocValuesField::new + ); } @Override diff --git a/x-pack/plugin/wildcard/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/wildcard/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..4dbb38f390287 --- /dev/null +++ b/x-pack/plugin/wildcard/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.wildcard.WildcardPainlessExtension diff --git a/x-pack/plugin/wildcard/src/main/resources/org/elasticsearch/xpack/wildcard/org.elasticsearch.xpack.wildcard.txt b/x-pack/plugin/wildcard/src/main/resources/org/elasticsearch/xpack/wildcard/org.elasticsearch.xpack.wildcard.txt new file mode 100644 index 0000000000000..4e6fccbbabd5b --- /dev/null +++ b/x-pack/plugin/wildcard/src/main/resources/org/elasticsearch/xpack/wildcard/org.elasticsearch.xpack.wildcard.txt @@ -0,0 +1,12 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +class org.elasticsearch.xpack.wildcard.WildcardDocValuesField @dynamic_type { + String get(String) + String get(int, String) +} diff --git a/x-pack/plugin/wildcard/src/yamlRestTest/java/org/elasticsearch/xpack/wildcard/WildcardClientYamlTestSuiteIT.java b/x-pack/plugin/wildcard/src/yamlRestTest/java/org/elasticsearch/xpack/wildcard/WildcardClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..61eb0c8b0de3e --- /dev/null +++ b/x-pack/plugin/wildcard/src/yamlRestTest/java/org/elasticsearch/xpack/wildcard/WildcardClientYamlTestSuiteIT.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.wildcard; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +/** Runs yaml rest tests */ +public class WildcardClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public WildcardClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml new file mode 100644 index 0000000000000..c16887f229b97 --- /dev/null +++ b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml @@ -0,0 +1,42 @@ +setup: + - do: + indices.create: + index: test + body: + mappings: + properties: + test_wc: + type: wildcard + + - do: + bulk: + index: test + refresh: true + body: | + { "index": {"_id" : "1"} } + { "test_wc": "abc" } + { "index": {"_id" : "2"} } + { "test_wc": "adc" } + +--- +"Constant Keyword Fields API": + - do: + search: + index: test + body: + query: + wildcard: + test_wc: + value: "a*c" + script_fields: + constOne: + script: + source: "/* avoid stash */ $('test_wc', 'dne')" + constTwo: + script: + source: "field('test_wc').get(0, 'dne')" + + - match: { hits.hits.0.fields.constOne.0: "abc" } + - match: { hits.hits.0.fields.constTwo.0: "abc" } + - match: { hits.hits.1.fields.constOne.0: "adc" } + - match: { hits.hits.1.fields.constTwo.0: "adc" } From a2e4610cd89d7cae8bc95e92786b8a4203ed309b Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 18 Jan 2022 12:00:27 -0800 Subject: [PATCH 2/4] Update docs/changelog/82763.yaml --- docs/changelog/82763.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/82763.yaml diff --git a/docs/changelog/82763.yaml b/docs/changelog/82763.yaml new file mode 100644 index 0000000000000..080a15f1dac98 --- /dev/null +++ b/docs/changelog/82763.yaml @@ -0,0 +1,5 @@ +pr: 82763 +summary: Make wildcard accessible from the scripting field API +area: Infra/Scripting +type: enhancement +issues: [] From b5a44357e16cd92d84f57ad5bcac4f935b56148d Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 18 Jan 2022 12:02:27 -0800 Subject: [PATCH 3/4] fix naming in tests --- .../resources/rest-api-spec/test/10_script_values.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml index c16887f229b97..290ed763881af 100644 --- a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml +++ b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml @@ -19,7 +19,7 @@ setup: { "test_wc": "adc" } --- -"Constant Keyword Fields API": +"Wildcard Fields API": - do: search: index: test From 5ab373f12078744376c3289a70e568bf8a64a757 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 18 Jan 2022 13:23:24 -0800 Subject: [PATCH 4/4] response to pr comments --- .../wildcard/org.elasticsearch.xpack.wildcard.txt | 2 -- .../rest-api-spec/test/10_script_values.yml | 15 +++++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/wildcard/src/main/resources/org/elasticsearch/xpack/wildcard/org.elasticsearch.xpack.wildcard.txt b/x-pack/plugin/wildcard/src/main/resources/org/elasticsearch/xpack/wildcard/org.elasticsearch.xpack.wildcard.txt index 4e6fccbbabd5b..73a155487c659 100644 --- a/x-pack/plugin/wildcard/src/main/resources/org/elasticsearch/xpack/wildcard/org.elasticsearch.xpack.wildcard.txt +++ b/x-pack/plugin/wildcard/src/main/resources/org/elasticsearch/xpack/wildcard/org.elasticsearch.xpack.wildcard.txt @@ -7,6 +7,4 @@ # class org.elasticsearch.xpack.wildcard.WildcardDocValuesField @dynamic_type { - String get(String) - String get(int, String) } diff --git a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml index 290ed763881af..c844b74565e08 100644 --- a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml +++ b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml @@ -3,6 +3,8 @@ setup: indices.create: index: test body: + settings: + number_of_shards: 1 mappings: properties: test_wc: @@ -16,7 +18,7 @@ setup: { "index": {"_id" : "1"} } { "test_wc": "abc" } { "index": {"_id" : "2"} } - { "test_wc": "adc" } + { "test_wc": ["adc", "aec"] } --- "Wildcard Fields API": @@ -34,9 +36,14 @@ setup: source: "/* avoid stash */ $('test_wc', 'dne')" constTwo: script: - source: "field('test_wc').get(0, 'dne')" + source: "field('test_wc').get(0, 'dne') + field('test_wc').get(1, 'xyz')" + constThree: + script: + source: "String s = ''; for (String z : field('test_wc')) s += z; return s" - match: { hits.hits.0.fields.constOne.0: "abc" } - - match: { hits.hits.0.fields.constTwo.0: "abc" } + - match: { hits.hits.0.fields.constTwo.0: "abcxyz" } + - match: { hits.hits.0.fields.constThree.0: "abc" } - match: { hits.hits.1.fields.constOne.0: "adc" } - - match: { hits.hits.1.fields.constTwo.0: "adc" } + - match: { hits.hits.1.fields.constTwo.0: "adcaec" } + - match: { hits.hits.1.fields.constThree.0: "adcaec" }