Script: fields API for flattened mapped type (#82590)

* Script: fields API for flattened mapped type

The flattened field type exposes all leaf values
as keyword doc values.  Additionally, specific keys
are available via object dot notation.

For example:

```
{
  "flat": {
    "abc": "bar",
    "def": "foo",
    "hij": {
      "lmn": "pqr",
      "stu": 123
    }
  }
}

field('flat').get('default') // returns 123
field('flat.abc').get('default') // returns bar
```

API:
* `iterator()`
* `get(String default)`
* `get(String default, int index)`

Refs: #79105

Author: stu-elastic

docs/changelog/82590.yaml

@@ -0,0 +1,5 @@
+pr: 82590
+summary: "Script: fields API for flattened mapped type"
+area: Infra/Scripting
+type: enhancement
+issues: []

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt

@@ -92,11 +92,19 @@ class org.elasticsearch.script.field.DateNanosDocValuesField @dynamic_type {
   ZonedDateTime get(int, ZonedDateTime)
 }
 
-class org.elasticsearch.script.field.KeywordDocValuesField @dynamic_type {
+class org.elasticsearch.script.field.AbstractKeywordDocValuesField @dynamic_type {
   String get(String)
   String get(int, String)
 }
 
+# subclass of AbstractKeywordDocValuesField
+class org.elasticsearch.script.field.KeywordDocValuesField @dynamic_type {
+}
+
+# subclass of AbstractKeywordDocValuesField
+class org.elasticsearch.script.field.FlattenedDocValuesField @dynamic_type {
+}
+
 class org.elasticsearch.script.field.GeoPointDocValuesField @dynamic_type {
   GeoPoint get(GeoPoint)
   GeoPoint get(int, GeoPoint)

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml

@@ -41,6 +41,8 @@ setup:
                             analyzer: standard
                         rank:
                             type: integer
+                        flattended:
+                            type: flattened
 
 
     - do:
@@ -1531,7 +1533,7 @@ setup:
 
   - do:
       index:
-        index: test
+        index: versiontest
         id: 3000
         version: 50
         version_type: external
@@ -1543,6 +1545,7 @@ setup:
 
   - do:
       search:
+        index: versiontest
         rest_total_hits_as_int: true
         body:
           query: { term: { _id: 3000 } }
@@ -1554,11 +1557,11 @@ setup:
               script:
                 source: "field('_seq_no').get(10000)"
   - match: { hits.hits.0.fields.ver.0: 50 }
-  - match: { hits.hits.0.fields.seq.0: 3 }
+  - match: { hits.hits.0.fields.seq.0: 0 }
 
   - do:
       index:
-        index: test
+        index: versiontest
         id: 3000
         version: 60
         version_type: external
@@ -1570,7 +1573,7 @@ setup:
   - do:
       catch:      conflict
       index:
-        index: test
+        index: versiontest
         id: 3000
         version: 55
         version_type: external
@@ -1581,6 +1584,7 @@ setup:
 
   - do:
       search:
+        index: versiontest
         rest_total_hits_as_int: true
         body:
           query: { term: { _id: 3000 } }
@@ -1592,4 +1596,103 @@ setup:
               script:
                 source: "field('_seq_no').get(10000)"
   - match: { hits.hits.0.fields.ver.0: 60 }
-  - match: { hits.hits.0.fields.seq.0: 4 }
+  - match: { hits.hits.0.fields.seq.0: 1 }
+
+---
+"flattened fields api":
+  - do:
+      indices.create:
+        index: flatindex
+        body:
+          settings:
+            number_of_shards: 1
+          mappings:
+            properties:
+              rank:
+                type: integer
+              flattened:
+                type: flattened
+
+  - do:
+      index:
+        index: flatindex
+        id: 40
+        body:
+          rank: 1
+          flattened:
+            top: 123
+            dict:
+              abc: def
+              hij: lmn
+            list: [true]
+
+  - do:
+      index:
+        index: flatindex
+        id: 41
+        body:
+          rank: 2
+          flattened:
+            top2: 876
+            dict2:
+              abc: def
+              opq: rst
+              hij: lmn
+              uvx: wyz
+            list2: [789, 1011, 1213]
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: flatindex
+        rest_total_hits_as_int: true
+        body:
+          sort: [ { rank: asc } ]
+          script_fields:
+            f_root:
+              script:
+                source: "field('flattened').get('dne')"
+            f_root_index:
+              script:
+                source: "field('flattened').get(2, 'dne')"
+            f_top:
+              script:
+                source: "field('flattened.top').get('dne')"
+            f_top2:
+              script:
+                source: "field('flattened.top2').get('dne')"
+            f_dict:
+              script:
+                source: "field('flattened.dict.abc').get('dne')"
+            f_dict2:
+              script:
+                source: "field('flattened.dict2.uvx').get('dne')"
+            f_list:
+              script:
+                source: "field('flattened.list').get('dne')"
+            f_list2:
+              script:
+                source: "field('flattened.list2').get(2, 'dne')"
+            all:
+              script:
+                source: "String all = ''; for (String value : field('flattened')) { all += value } all"
+  - match: { hits.hits.0.fields.f_root.0: "123" }
+  - match: { hits.hits.0.fields.f_root_index.0: "lmn" }
+  - match: { hits.hits.0.fields.f_top.0: "123" }
+  - match: { hits.hits.0.fields.f_top2.0: "dne" }
+  - match: { hits.hits.0.fields.f_dict.0: "def" }
+  - match: { hits.hits.0.fields.f_dict2.0: "dne" }
+  - match: { hits.hits.0.fields.f_list.0: "true" }
+  - match: { hits.hits.0.fields.f_list2.0: "dne" }
+  - match: { hits.hits.0.fields.all.0: "123deflmntrue" }
+  - match: { hits.hits.1.fields.f_root.0: "1011" }
+  - match: { hits.hits.1.fields.f_root_index.0: "789" }
+  - match: { hits.hits.1.fields.f_top.0: "dne" }
+  - match: { hits.hits.1.fields.f_top2.0: "876" }
+  - match: { hits.hits.1.fields.f_dict.0: "dne" }
+  - match: { hits.hits.1.fields.f_dict2.0: "wyz" }
+  - match: { hits.hits.1.fields.f_list.0: "dne" }
+  - match: { hits.hits.1.fields.f_list2.0: "789" }
+  - match: { hits.hits.1.fields.all.0: "10111213789876deflmnrstwyz" }

server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java

@@ -41,7 +41,6 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
 import org.elasticsearch.index.mapper.DocumentParserContext;
@@ -58,7 +57,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.script.field.DelegateDocValuesField;
+import org.elasticsearch.script.field.FlattenedDocValuesField;
 import org.elasticsearch.script.field.ToScriptField;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
@@ -199,11 +198,7 @@ public FlattenedFieldMapper build(MapperBuilderContext context) {
                 hasDocValues.get(),
                 meta.get(),
                 splitQueriesOnWhitespace.get(),
-                eagerGlobalOrdinals.get(),
-                (dv, n) -> new DelegateDocValuesField(
-                    new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))),
-                    n
-                )
+                eagerGlobalOrdinals.get()
             );
             return new FlattenedFieldMapper(name, ft, this);
         }
@@ -367,14 +362,7 @@ public BytesRef indexedValueForSearch(Object value) {
         @Override
         public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
             failIfNoDocValues();
-            return new KeyedFlattenedFieldData.Builder(
-                name(),
-                key,
-                (dv, n) -> new DelegateDocValuesField(
-                    new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))),
-                    n
-                )
-            );
+            return new KeyedFlattenedFieldData.Builder(name(), key, (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n));
         }
 
         @Override
@@ -603,16 +591,14 @@ public IndexFieldData<?> build(IndexFieldDataCache cache, CircuitBreakerService
     public static final class RootFlattenedFieldType extends StringFieldType implements DynamicFieldType {
         private final boolean splitQueriesOnWhitespace;
         private final boolean eagerGlobalOrdinals;
-        private final ToScriptField<SortedSetDocValues> toScriptField;
 
         public RootFlattenedFieldType(
             String name,
             boolean indexed,
             boolean hasDocValues,
             Map<String, String> meta,
             boolean splitQueriesOnWhitespace,
-            boolean eagerGlobalOrdinals,
-            ToScriptField<SortedSetDocValues> toScriptField
+            boolean eagerGlobalOrdinals
         ) {
             super(
                 name,
@@ -624,7 +610,6 @@ public RootFlattenedFieldType(
             );
             this.splitQueriesOnWhitespace = splitQueriesOnWhitespace;
             this.eagerGlobalOrdinals = eagerGlobalOrdinals;
-            this.toScriptField = toScriptField;
         }
 
         @Override
@@ -657,10 +642,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S
             return new SortedSetOrdinalsIndexFieldData.Builder(
                 name(),
                 CoreValuesSourceType.KEYWORD,
-                (dv, n) -> new DelegateDocValuesField(
-                    new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))),
-                    n
-                )
+                (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n)
             );
         }
 

server/src/main/java/org/elasticsearch/script/field/FlattenedDocValuesField.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script.field;
+
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+
+public class FlattenedDocValuesField extends AbstractKeywordDocValuesField {
+    public FlattenedDocValuesField(SortedBinaryDocValues input, String name) {
+        super(input, name);
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java

@@ -8,7 +8,6 @@
 
 package org.elasticsearch.index.mapper.flattened;
 
-import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.FuzzyQuery;
@@ -21,27 +20,19 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
 import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.index.fielddata.FieldData;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType;
-import org.elasticsearch.script.field.DelegateDocValuesField;
-import org.elasticsearch.script.field.ToScriptField;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
 public class RootFlattenedFieldTypeTests extends FieldTypeTestCase {
-    private static final ToScriptField<SortedSetDocValues> MOCK_TO_SCRIPT_FIELD = (dv, n) -> new DelegateDocValuesField(
-        new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))),
-        n
-    );
 
     private static RootFlattenedFieldType createDefaultFieldType() {
-        return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, MOCK_TO_SCRIPT_FIELD);
+        return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false);
     }
 
     public void testValueForDisplay() {
@@ -61,40 +52,16 @@ public void testTermQuery() {
         expected = AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "Value"));
         assertEquals(expected, ft.termQueryCaseInsensitive("Value", null));
 
-        RootFlattenedFieldType unsearchable = new RootFlattenedFieldType(
-            "field",
-            false,
-            true,
-            Collections.emptyMap(),
-            false,
-            false,
-            MOCK_TO_SCRIPT_FIELD
-        );
+        RootFlattenedFieldType unsearchable = new RootFlattenedFieldType("field", false, true, Collections.emptyMap(), false, false);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null));
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 
     public void testExistsQuery() {
-        RootFlattenedFieldType ft = new RootFlattenedFieldType(
-            "field",
-            true,
-            false,
-            Collections.emptyMap(),
-            false,
-            false,
-            MOCK_TO_SCRIPT_FIELD
-        );
+        RootFlattenedFieldType ft = new RootFlattenedFieldType("field", true, false, Collections.emptyMap(), false, false);
         assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null));
 
-        RootFlattenedFieldType withDv = new RootFlattenedFieldType(
-            "field",
-            true,
-            true,
-            Collections.emptyMap(),
-            false,
-            false,
-            MOCK_TO_SCRIPT_FIELD
-        );
+        RootFlattenedFieldType withDv = new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false);
         assertEquals(new DocValuesFieldExistsQuery("field"), withDv.existsQuery(null));
     }
 

x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt

@@ -6,7 +6,6 @@
 # Side Public License, v 1.
 #
 
+# subclass of AbstractKeywordDocValuesField
 class org.elasticsearch.xpack.constantkeyword.ConstantKeywordDocValuesField @dynamic_type {
-  String get(String)
-  String get(int, String)
 }