Add script parameter to long and double field mappers (#69531) (#71105)

This commit adds a script parameter to long and double fields that makes
it possible to calculate a value for these fields at index time. It uses the same
script context as the equivalent runtime fields, and allows for multiple index-time
scripted fields to cross-refer while still checking for indirection loops.

Author: romseygeek

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -81,7 +81,7 @@ public class ScriptScoreBenchmark {
     private final Map<String, MappedFieldType> fieldTypes = org.elasticsearch.common.collect.Map.ofEntries(
         org.elasticsearch.common.collect.Map.entry(
             "n",
-            new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, org.elasticsearch.common.collect.Map.of())
+            new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, org.elasticsearch.common.collect.Map.of(), null)
         )
     );
     private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None();

docs/reference/mapping/types/numeric.asciidoc

@@ -117,6 +117,7 @@ The following parameters are accepted by numeric types:
 
     Try to convert strings to numbers and truncate fractions for integers.
     Accepts `true` (default) and `false`. Not applicable for `unsigned_long`.
+    Note that this cannot be set if the `script` parameter is used.
 
 <<mapping-boost,`boost`>>::
 
@@ -132,7 +133,8 @@ The following parameters are accepted by numeric types:
 <<ignore-malformed,`ignore_malformed`>>::
 
     If `true`, malformed numbers are ignored. If `false` (default), malformed
-    numbers throw an exception and reject the whole document.
+    numbers throw an exception and reject the whole document.  Note that this
+    cannot be set if the `script` parameter is used.
 
 <<mapping-index,`index`>>::
 
@@ -142,7 +144,26 @@ The following parameters are accepted by numeric types:
 
     Accepts a numeric value of the same `type` as the field which is
     substituted for any explicit `null` values.  Defaults to `null`, which
-    means the field is treated as missing.
+    means the field is treated as missing.  Note that this cannot be set
+    if the `script` parameter is used.
+
+`on_script_error`::
+
+    Defines what to do if the script defined by the `script` parameter
+    throws an error at indexing time.  Accepts `reject` (default), which
+    will cause the entire document to be rejected, and `ignore`, which
+    will register the field in the document's
+    <<mapping-ignored-field,`_ignored`>> metadata field and continue
+    indexing.  This parameter can only be set if the `script` field is
+    also set.
+
+`script`::
+
+    If this parameter is set, then the field will index values generated
+    by this script, rather than reading the values directly from the
+    source.  Scripts are in the same format as their
+    <<runtime-mapping-fields,runtime equivalent>>. Scripts can only be
+    configured on `long` and `double` field types.
 
 <<mapping-store,`store`>>::
 

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java

@@ -77,7 +77,7 @@ static class TokenCountFieldType extends NumberFieldMapper.NumberFieldType {
 
         TokenCountFieldType(String name, boolean isSearchable, boolean isStored,
                             boolean hasDocValues, Number nullValue, Map<String, String> meta) {
-            super(name, NumberFieldMapper.NumberType.INTEGER, isSearchable, isStored, hasDocValues, false, nullValue, meta);
+            super(name, NumberFieldMapper.NumberType.INTEGER, isSearchable, isStored, hasDocValues, false, nullValue, meta, null);
         }
 
         @Override

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java

@@ -57,6 +57,11 @@ protected Collection<? extends Plugin> getPlugins() {
         return List.of(new MapperExtrasPlugin());
     }
 
+    @Override
+    protected boolean allowsStore() {
+        return false;
+    }
+
     static int getFrequency(TokenStream tk) throws IOException {
         TermFrequencyAttribute freqAttribute = tk.addAttribute(TermFrequencyAttribute.class);
         tk.reset();

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java

@@ -53,6 +53,11 @@ protected boolean supportsMeta() {
         return false;
     }
 
+    @Override
+    protected boolean allowsStore() {
+        return false;
+    }
+
     public void testDefaults() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
         assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString());

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/23_long_calculated_at_index.yml

@@ -0,0 +1,152 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: sensor
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              timestamp:
+                type: date
+              temperature:
+                type: long
+              voltage:
+                type: double
+              node:
+                type: keyword
+              voltage_times_ten:
+                type: long
+                script:
+                  source: |
+                   for (double v : doc['voltage']) {
+                     emit((long)(v * params.multiplier));
+                   }
+                  params:
+                    multiplier: 10
+              voltage_times_ten_no_dv:
+                type: long
+                doc_values: false
+                script:
+                  source: |
+                    for (double v : doc['voltage']) {
+                      emit((long)(v * params.multiplier));
+                    }
+                  params:
+                    multiplier: 10
+              # test multiple values
+              temperature_digits:
+                type: long
+                script:
+                  source: |
+                    for (long temperature : doc['temperature']) {
+                      long t = temperature;
+                      while (t != 0) {
+                        emit(t % 10);
+                        t /= 10;
+                      }
+                    }
+
+  - do:
+      bulk:
+        index: sensor
+        refresh: true
+        body: |
+          {"index":{}}
+          {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"}
+          {"index":{}}
+          {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"}
+          {"index":{}}
+          {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"}
+          {"index":{}}
+          {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"}
+          {"index":{}}
+          {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
+          {"index":{}}
+          {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
+
+---
+"get mapping":
+  - do:
+      indices.get_mapping:
+        index: sensor
+  - match: {sensor.mappings.properties.voltage_times_ten.type: long }
+  - match:
+      sensor.mappings.properties.voltage_times_ten.script.source: |
+        for (double v : doc['voltage']) {
+          emit((long)(v * params.multiplier));
+        }
+  - match: {sensor.mappings.properties.voltage_times_ten.script.params: {multiplier: 10} }
+  - match: {sensor.mappings.properties.voltage_times_ten.script.lang: painless }
+
+---
+"fetch fields":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          fields:
+            - voltage_times_ten
+            - voltage_times_ten_no_dv
+            - temperature_digits
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.voltage_times_ten: [40] }
+  - match: {hits.hits.0.fields.temperature_digits: [2, 0, 2] }
+  - match: {hits.hits.0.fields.voltage_times_ten: [40] }
+  - match: {hits.hits.0.fields.voltage_times_ten_no_dv: [40] }
+  - match: {hits.hits.1.fields.voltage_times_ten: [42] }
+  - match: {hits.hits.2.fields.voltage_times_ten: [56] }
+  - match: {hits.hits.3.fields.voltage_times_ten: [51] }
+  - match: {hits.hits.4.fields.voltage_times_ten: [58] }
+  - match: {hits.hits.5.fields.voltage_times_ten: [52] }
+
+---
+"docvalue_fields":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          docvalue_fields:
+            - voltage_times_ten
+            - temperature_digits
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.voltage_times_ten: [40] }
+  - match: {hits.hits.0.fields.temperature_digits: [0, 2, 2] }
+  - match: {hits.hits.0.fields.voltage_times_ten: [40] }
+  - match: {hits.hits.1.fields.voltage_times_ten: [42] }
+  - match: {hits.hits.2.fields.voltage_times_ten: [56] }
+  - match: {hits.hits.3.fields.voltage_times_ten: [51] }
+  - match: {hits.hits.4.fields.voltage_times_ten: [58] }
+  - match: {hits.hits.5.fields.voltage_times_ten: [52] }
+
+---
+"terms agg":
+  - do:
+      search:
+        index: sensor
+        body:
+          aggs:
+            v10:
+              terms:
+                field: voltage_times_ten
+  - match: {hits.total.value: 6}
+  - match: {aggregations.v10.buckets.0.key: 40.0}
+  - match: {aggregations.v10.buckets.0.doc_count: 1}
+  - match: {aggregations.v10.buckets.1.key: 42.0}
+  - match: {aggregations.v10.buckets.1.doc_count: 1}
+
+---
+"term query":
+  - do:
+      search:
+        index: sensor
+        body:
+          query:
+            term:
+              voltage_times_ten: 58
+  - match: {hits.total.value: 1}
+  - match: {hits.hits.0._source.voltage: 5.8}