Add BinaryDocValuesField to replace BytesRef (ScriptDocValues) (#79760)

This change creates the classes required for the scripting fields API to provide a binary field 
composed of doc values using BytesRef as the representation returned to the user as a value.

Author: jdconrad

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt

@@ -8,18 +8,25 @@
 
 # The whitelist for the fields api
 
-# API
 class org.elasticsearch.script.field.Field @dynamic_type {
   String getName()
   boolean isEmpty()
   int size()
 }
 
+class org.elasticsearch.script.field.EmptyField @dynamic_type {
+  def get(def)
+  def get(int, def)
+}
+
 class org.elasticsearch.script.DocBasedScript {
   org.elasticsearch.script.field.Field field(String)
 }
 
 class org.elasticsearch.script.field.DelegateDocValuesField @dynamic_type {
-  def getValue(def)
-  List getValues()
 }
+
+class org.elasticsearch.script.field.BinaryDocValuesField @dynamic_type {
+  ByteBuffer get(ByteBuffer)
+  ByteBuffer get(int, ByteBuffer)
+}

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/20_scriptfield.yml

@@ -157,31 +157,4 @@ setup:
     - match: { error.failed_shards.0.reason.type: "script_exception" }
     - match: { error.failed_shards.0.reason.reason: "compile error" }
 
----
-"Scripted Field with error accessing an unsupported field via the script fields api":
-  - do:
-      catch: bad_request
-      search:
-        rest_total_hits_as_int: true
-        body:
-          script_fields:
-            bar:
-              script:
-                source: "field('foo').getValue('')"
-
-  - match: { error.failed_shards.0.reason.caused_by.type: "unsupported_operation_exception" }
-  - match: { error.failed_shards.0.reason.caused_by.reason: "field [foo] is not supported through the fields api, use [doc] instead"}
-
-  - do:
-      catch: bad_request
-      search:
-        rest_total_hits_as_int: true
-        body:
-          script_fields:
-            bar:
-              script:
-                source: "field('foo').getValues()"
-
-  - match: { error.failed_shards.0.reason.caused_by.type: "unsupported_operation_exception" }
-  - match: { error.failed_shards.0.reason.caused_by.reason: "field [foo] is not supported through the fields api, use [doc] instead" }
 

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml

@@ -440,3 +440,25 @@ setup:
                         script:
                             source: "doc['token_count'].value"
     - match: { hits.hits.0.fields.field.0: 5 }
+
+---
+"empty":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          script_fields:
+            field:
+              script:
+                source: "int value = field('dne').get(1); value"
+  - match: { hits.hits.0.fields.field.0: 1}
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          script_fields:
+            field:
+              script:
+                source: "int value = field('dne').get(1, 1); value"
+  - match: { hits.hits.0.fields.field.0: 1 }

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml

@@ -20,7 +20,30 @@
             index: test
             id: 1
             body:
-                binary: U29tZSBiaW5hcnkgYmxvYg==
+                binary: "U29tZSBiaW5hcnkgYmxvYg=="
+
+    - do:
+        #set the header so we won't randomize it
+        headers:
+          Content-Type: application/json
+        index:
+          index: test
+          id: 2
+          body:
+            binary: [
+              "U29tZSBiaW5hcnkgYmxvYg==",
+              "MTIzNA==",
+              "dGVzdA=="
+            ]
+
+    - do:
+        #set the header so we won't randomize it
+        headers:
+          Content-Type: application/json
+        index:
+          index: test
+          id: 3
+          body: {}
 
     - do:
         indices.refresh: {}
@@ -31,9 +54,55 @@
                 script_fields:
                     field1:
                         script:
-                            source: "doc['binary'].get(0).utf8ToString()"
+                            source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].get(0).utf8ToString()"
                     field2:
                         script:
-                            source: "doc['binary'].value.utf8ToString()"
+                            source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].value.utf8ToString()"
     - match: { hits.hits.0.fields.field1.0: "Some binary blob" }
     - match: { hits.hits.0.fields.field2.0: "Some binary blob" }
+
+    - do:
+        search:
+          body:
+            script_fields:
+              field1:
+                script:
+                  source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.get(0)"
+              field2:
+                script:
+                  source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.get(0)"
+              field3:
+                script:
+                  source: "int total = 0; for (value in field('binary')) {total += value.get(0)} total"
+    - match: { hits.hits.0.fields.field1.0: 83 }
+    - match: { hits.hits.0.fields.field2.0: 83 }
+    - match: { hits.hits.0.fields.field3.0: 83 }
+    - match: { hits.hits.1.fields.field1.0: 49 }
+    - match: { hits.hits.1.fields.field2.0: 49 }
+    - match: { hits.hits.1.fields.field3.0: 248 }
+    - match: { hits.hits.2.fields.field1.0: -1 }
+    - match: { hits.hits.2.fields.field2.0: -1 }
+    - match: { hits.hits.2.fields.field3.0: 0 }
+
+    - do:
+        search:
+          body:
+            script_fields:
+              field1:
+                script:
+                  source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.limit()"
+              field2:
+                script:
+                  source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.limit()"
+              field3:
+                script:
+                  source: "int total = 0; for (ByteBuffer value : field('binary')) {total += value.limit()} total"
+    - match: { hits.hits.0.fields.field1.0: 16 }
+    - match: { hits.hits.0.fields.field2.0: 16 }
+    - match: { hits.hits.0.fields.field3.0: 16 }
+    - match: { hits.hits.1.fields.field1.0: 4 }
+    - match: { hits.hits.1.fields.field2.0: 4 }
+    - match: { hits.hits.1.fields.field3.0: 24 }
+    - match: { hits.hits.2.fields.field1.0: -1 }
+    - match: { hits.hits.2.fields.field2.0: -1 }
+    - match: { hits.hits.2.fields.field3.0: 0 }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -527,7 +527,7 @@ private static class ScaledFloatLeafFieldData implements LeafNumericFieldData {
         }
 
         @Override
-        public DocValuesField getScriptField(String name) {
+        public DocValuesField<?> getScriptField(String name) {
             return new DelegateDocValuesField(new ScriptDocValues.Doubles(getDoubleValues()), name);
         }
 

server/src/main/java/org/elasticsearch/index/fielddata/IpScriptFieldData.java

@@ -52,7 +52,7 @@ public BinaryScriptLeafFieldData loadDirect(LeafReaderContext context) throws Ex
         IpFieldScript script = leafFactory.newInstance(context);
         return new BinaryScriptLeafFieldData() {
             @Override
-            public DocValuesField getScriptField(String name) {
+            public DocValuesField<?> getScriptField(String name) {
                 return new DelegateDocValuesField(new IpScriptDocValues(getBytesValues()), name);
             }
 

server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java

@@ -23,7 +23,7 @@ public interface LeafFieldData extends Accountable, Releasable {
     /**
      * Returns an {@code Field} for use in accessing field values in scripting.
      */
-    DocValuesField getScriptField(String name);
+    DocValuesField<?> getScriptField(String name);
 
     /**
      * Return a String representation of the values.

server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.geometry.utils.Geohash;
+import org.elasticsearch.script.field.BinaryDocValuesField;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -585,30 +586,33 @@ public final String getValue() {
         }
     }
 
-    public static final class BytesRefs extends BinaryScriptDocValues<BytesRef> {
+    public static final class BytesRefs extends ScriptDocValues<BytesRef> {
 
-        public BytesRefs(SortedBinaryDocValues in) {
-            super(in);
+        private final BinaryDocValuesField binaryDocValuesField;
+
+        public BytesRefs(BinaryDocValuesField binaryDocValuesField) {
+            this.binaryDocValuesField = binaryDocValuesField;
         }
 
         @Override
-        public BytesRef get(int index) {
-            if (count == 0) {
-                throw new IllegalStateException(
-                    "A document doesn't have a value for a field! "
-                        + "Use doc[<field>].size()==0 to check if a document is missing a field!"
-                );
-            }
-            /**
-             * We need to make a copy here because {@link BinaryScriptDocValues} might reuse the
-             * returned value and the same instance might be used to
-             * return values from multiple documents.
-             **/
-            return values[index].toBytesRef();
+        public void setNextDocId(int docId) throws IOException {
+            throw new UnsupportedOperationException();
         }
 
         public BytesRef getValue() {
+            throwIfEmpty();
             return get(0);
         }
+
+        @Override
+        public BytesRef get(int index) {
+            throwIfEmpty();
+            return binaryDocValuesField.getInternal(index);
+        }
+
+        @Override
+        public int size() {
+            return binaryDocValuesField.size();
+        }
     }
 }

server/src/main/java/org/elasticsearch/index/fielddata/StringScriptFieldData.java

@@ -44,7 +44,7 @@ public BinaryScriptLeafFieldData loadDirect(LeafReaderContext context) throws Ex
         StringFieldScript script = leafFactory.newInstance(context);
         return new BinaryScriptLeafFieldData() {
             @Override
-            public DocValuesField getScriptField(String name) {
+            public DocValuesField<?> getScriptField(String name) {
                 return new DelegateDocValuesField(new ScriptDocValues.Strings(getBytesValues()), name);
             }
 

server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafGeoPointFieldData.java

@@ -27,7 +27,7 @@ public final SortedBinaryDocValues getBytesValues() {
     }
 
     @Override
-    public final DocValuesField getScriptField(String name) {
+    public final DocValuesField<?> getScriptField(String name) {
         return new DelegateDocValuesField(new ScriptDocValues.GeoPoints(getGeoPointValues()), name);
     }