From 3fa6bb719f78467635979cb7c998eb35d978d6d7 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 13 Jan 2022 16:32:33 -0600 Subject: [PATCH 1/3] Script: fields API for flattened mapped type The flattened field type exposes all leaf values as keyword doc values. Additionally, specific keys are available via object dot notation. For example: ``` { "flat": { "abc": "bar", "def": "foo", "hij": { "lmn": "pqr", "stu": 123 } } } field('flat').get('default') // returns 123 field('flat.abc').get('default') // returns bar ``` API: * `iterator()` * `get(String default)` * `get(String default, int index)` Refs: #79105 --- .../org.elasticsearch.script.fields.txt | 10 +- .../test/painless/50_script_doc_values.yml | 113 +++++++++++++++++- .../flattened/FlattenedFieldMapper.java | 24 +--- .../script/field/FlattenedDocValuesField.java | 17 +++ .../RootFlattenedFieldTypeTests.java | 41 +------ ...rg.elasticsearch.xpack.constantkeyword.txt | 3 +- 6 files changed, 144 insertions(+), 64 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/script/field/FlattenedDocValuesField.java diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt index 6a9fe2da78c88..699221acbcf3f 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt @@ -92,11 +92,19 @@ class org.elasticsearch.script.field.DateNanosDocValuesField @dynamic_type { ZonedDateTime get(int, ZonedDateTime) } -class org.elasticsearch.script.field.KeywordDocValuesField @dynamic_type { +class org.elasticsearch.script.field.AbstractKeywordDocValuesField @dynamic_type { String get(String) String get(int, String) } +# subclass of AbstractKeywordDocValuesField +class org.elasticsearch.script.field.KeywordDocValuesField @dynamic_type { +} + +# subclass of AbstractKeywordDocValuesField +class org.elasticsearch.script.field.FlattenedDocValuesField @dynamic_type { +} + class org.elasticsearch.script.field.GeoPointDocValuesField @dynamic_type { GeoPoint get(GeoPoint) GeoPoint get(int, GeoPoint) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 79f06f6a23ab0..9ddc05d6675c2 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -41,6 +41,8 @@ setup: analyzer: standard rank: type: integer + flattended: + type: flattened - do: @@ -1531,7 +1533,7 @@ setup: - do: index: - index: test + index: versiontest id: 3000 version: 50 version_type: external @@ -1543,6 +1545,7 @@ setup: - do: search: + index: versiontest rest_total_hits_as_int: true body: query: { term: { _id: 3000 } } @@ -1554,11 +1557,11 @@ setup: script: source: "field('_seq_no').get(10000)" - match: { hits.hits.0.fields.ver.0: 50 } - - match: { hits.hits.0.fields.seq.0: 3 } + - match: { hits.hits.0.fields.seq.0: 0 } - do: index: - index: test + index: versiontest id: 3000 version: 60 version_type: external @@ -1570,7 +1573,7 @@ setup: - do: catch: conflict index: - index: test + index: versiontest id: 3000 version: 55 version_type: external @@ -1581,6 +1584,7 @@ setup: - do: search: + index: versiontest rest_total_hits_as_int: true body: query: { term: { _id: 3000 } } @@ -1592,4 +1596,103 @@ setup: script: source: "field('_seq_no').get(10000)" - match: { hits.hits.0.fields.ver.0: 60 } - - match: { hits.hits.0.fields.seq.0: 4 } + - match: { hits.hits.0.fields.seq.0: 1 } + +--- +"flattened fields api": + - do: + indices.create: + index: flatindex + body: + settings: + number_of_shards: 1 + mappings: + properties: + rank: + type: integer + flattened: + type: flattened + + - do: + index: + index: flatindex + id: 40 + body: + rank: 1 + flattened: + top: 123 + dict: + abc: def + hij: lmn + list: [true] + + - do: + index: + index: flatindex + id: 41 + body: + rank: 2 + flattened: + top2: 876 + dict2: + abc: def + opq: rst + hij: lmn + uvx: wyz + list2: [789, 1011, 1213] + + - do: + indices.refresh: {} + + - do: + search: + index: flatindex + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + f_root: + script: + source: "field('flattened').get('dne')" + f_root_index: + script: + source: "field('flattened').get(2, 'dne')" + f_top: + script: + source: "field('flattened.top').get('dne')" + f_top2: + script: + source: "field('flattened.top2').get('dne')" + f_dict: + script: + source: "field('flattened.dict.abc').get('dne')" + f_dict2: + script: + source: "field('flattened.dict2.uvx').get('dne')" + f_list: + script: + source: "field('flattened.list').get('dne')" + f_list2: + script: + source: "field('flattened.list2').get(2, 'dne')" + all: + script: + source: "String all = ''; for (String value : field('flattened')) { all += value } all" + - match: { hits.hits.0.fields.f_root.0: "123" } + - match: { hits.hits.0.fields.f_root_index.0: "lmn" } + - match: { hits.hits.0.fields.f_top.0: "123" } + - match: { hits.hits.0.fields.f_top2.0: "dne" } + - match: { hits.hits.0.fields.f_dict.0: "def" } + - match: { hits.hits.0.fields.f_dict2.0: "dne" } + - match: { hits.hits.0.fields.f_list.0: "true" } + - match: { hits.hits.0.fields.f_list2.0: "dne" } + - match: { hits.hits.0.fields.all.0: "123deflmntrue" } + - match: { hits.hits.1.fields.f_root.0: "1011" } + - match: { hits.hits.1.fields.f_root_index.0: "789" } + - match: { hits.hits.1.fields.f_top.0: "dne" } + - match: { hits.hits.1.fields.f_top2.0: "876" } + - match: { hits.hits.1.fields.f_dict.0: "dne" } + - match: { hits.hits.1.fields.f_dict2.0: "wyz" } + - match: { hits.hits.1.fields.f_list.0: "dne" } + - match: { hits.hits.1.fields.f_list2.0: "789" } + - match: { hits.hits.1.fields.all.0: "10111213789876deflmnrstwyz" } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 5706f15129bc6..f43f8baa928d8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; @@ -58,7 +57,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.script.field.FlattenedDocValuesField; import org.elasticsearch.script.field.ToScriptField; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; @@ -199,11 +198,7 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { hasDocValues.get(), meta.get(), splitQueriesOnWhitespace.get(), - eagerGlobalOrdinals.get(), - (dv, n) -> new DelegateDocValuesField( - new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), - n - ) + eagerGlobalOrdinals.get() ); return new FlattenedFieldMapper(name, ft, this); } @@ -370,10 +365,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new KeyedFlattenedFieldData.Builder( name(), key, - (dv, n) -> new DelegateDocValuesField( - new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), - n - ) + (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n) ); } @@ -603,7 +595,6 @@ public IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService public static final class RootFlattenedFieldType extends StringFieldType implements DynamicFieldType { private final boolean splitQueriesOnWhitespace; private final boolean eagerGlobalOrdinals; - private final ToScriptField toScriptField; public RootFlattenedFieldType( String name, @@ -611,8 +602,7 @@ public RootFlattenedFieldType( boolean hasDocValues, Map meta, boolean splitQueriesOnWhitespace, - boolean eagerGlobalOrdinals, - ToScriptField toScriptField + boolean eagerGlobalOrdinals ) { super( name, @@ -624,7 +614,6 @@ public RootFlattenedFieldType( ); this.splitQueriesOnWhitespace = splitQueriesOnWhitespace; this.eagerGlobalOrdinals = eagerGlobalOrdinals; - this.toScriptField = toScriptField; } @Override @@ -652,10 +641,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new SortedSetOrdinalsIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - (dv, n) -> new DelegateDocValuesField( - new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), - n - ) + (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n) ); } diff --git a/server/src/main/java/org/elasticsearch/script/field/FlattenedDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/FlattenedDocValuesField.java new file mode 100644 index 0000000000000..80ff208fe4322 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/FlattenedDocValuesField.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; + +public class FlattenedDocValuesField extends AbstractKeywordDocValuesField { + public FlattenedDocValuesField(SortedBinaryDocValues input, String name) { + super(input, name); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index c1bbda7b32f44..7d6e85bb60b46 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.mapper.flattened; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.FuzzyQuery; @@ -21,13 +20,9 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.index.fielddata.FieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; -import org.elasticsearch.script.field.DelegateDocValuesField; -import org.elasticsearch.script.field.ToScriptField; import java.io.IOException; import java.util.Collections; @@ -35,13 +30,9 @@ import java.util.Map; public class RootFlattenedFieldTypeTests extends FieldTypeTestCase { - private static final ToScriptField MOCK_TO_SCRIPT_FIELD = (dv, n) -> new DelegateDocValuesField( - new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), - n - ); private static RootFlattenedFieldType createDefaultFieldType() { - return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, MOCK_TO_SCRIPT_FIELD); + return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false); } public void testValueForDisplay() { @@ -61,40 +52,16 @@ public void testTermQuery() { expected = AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "Value")); assertEquals(expected, ft.termQueryCaseInsensitive("Value", null)); - RootFlattenedFieldType unsearchable = new RootFlattenedFieldType( - "field", - false, - true, - Collections.emptyMap(), - false, - false, - MOCK_TO_SCRIPT_FIELD - ); + RootFlattenedFieldType unsearchable = new RootFlattenedFieldType("field", false, true, Collections.emptyMap(), false, false); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } public void testExistsQuery() { - RootFlattenedFieldType ft = new RootFlattenedFieldType( - "field", - true, - false, - Collections.emptyMap(), - false, - false, - MOCK_TO_SCRIPT_FIELD - ); + RootFlattenedFieldType ft = new RootFlattenedFieldType("field", true, false, Collections.emptyMap(), false, false); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null)); - RootFlattenedFieldType withDv = new RootFlattenedFieldType( - "field", - true, - true, - Collections.emptyMap(), - false, - false, - MOCK_TO_SCRIPT_FIELD - ); + RootFlattenedFieldType withDv = new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false); assertEquals(new DocValuesFieldExistsQuery("field"), withDv.existsQuery(null)); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt b/x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt index ff58c75b1ff8e..ec7cdcfe901e3 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt +++ b/x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt @@ -6,7 +6,6 @@ # Side Public License, v 1. # +# subclass of AbstractKeywordDocValuesField class org.elasticsearch.xpack.constantkeyword.ConstantKeywordDocValuesField @dynamic_type { - String get(String) - String get(int, String) } From 8baa26c2d6b1fc8a28dbb9710534ffae86092698 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 13 Jan 2022 16:36:17 -0600 Subject: [PATCH 2/3] Update docs/changelog/82590.yaml --- docs/changelog/82590.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/82590.yaml diff --git a/docs/changelog/82590.yaml b/docs/changelog/82590.yaml new file mode 100644 index 0000000000000..5ff9363197f92 --- /dev/null +++ b/docs/changelog/82590.yaml @@ -0,0 +1,5 @@ +pr: 82590 +summary: "Script: fields API for flattened mapped type" +area: Infra/Scripting +type: enhancement +issues: [] From 2173acab0b0163289f5d52fcebdb954ebfd3aa60 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 13 Jan 2022 16:50:28 -0600 Subject: [PATCH 3/3] spotless --- .../index/mapper/flattened/FlattenedFieldMapper.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index f43f8baa928d8..d9a151b761b9f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -362,11 +362,7 @@ public BytesRef indexedValueForSearch(Object value) { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); - return new KeyedFlattenedFieldData.Builder( - name(), - key, - (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n) - ); + return new KeyedFlattenedFieldData.Builder(name(), key, (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n)); } @Override