From 2178568a460ffe92fa0a18c1de4245d10a6a1163 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 5 Jan 2022 10:33:20 -0800 Subject: [PATCH 1/4] Add scaled float to the scripting fields API --- .../org.elasticsearch.script.fields.txt | 5 + .../test/painless/50_script_doc_values.yml | 40 ++++++ .../mapper/extras/ScaledFloatFieldMapper.java | 3 +- .../field/ScaledFloatDocValuesField.java | 120 ++++++++++++++++++ 4 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/org/elasticsearch/script/field/ScaledFloatDocValuesField.java diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt index 71c01ad7eddcb..ed7c68ef12163 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt @@ -56,6 +56,11 @@ class org.elasticsearch.script.field.FloatDocValuesField @dynamic_type { float get(int, float) } +class org.elasticsearch.script.field.ScaledFloatDocValuesField @dynamic_type { + double get(double) + double get(int, double) +} + # defaults are cast to byte, taking an int facilitates resolution with constants without casting class org.elasticsearch.script.field.ByteDocValuesField @dynamic_type { byte get(int) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 7adb2c87e6541..5f75036fdddc9 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -90,6 +90,7 @@ setup: byte: [16, 32, 64, 8, 4] double: [3.141592653588, 2.141592653587] float: [1.123, 2.234] + scaled_float: [-3.5, 2.5] - do: @@ -1340,6 +1341,45 @@ setup: source: "doc['scaled_float'].get(0)" - match: { hits.hits.0.fields.field.0: 3.14 } + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('scaled_float').get(0.0)" + - match: { hits.hits.0.fields.field.0: 3.14 } + - match: { hits.hits.1.fields.field.0: 0.0 } + - match: { hits.hits.2.fields.field.0: -3.5 } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "/* avoid stash */ $('scaled_float', 0.0)" + - match: { hits.hits.0.fields.field.0: 3.14 } + - match: { hits.hits.1.fields.field.0: 0.0 } + - match: { hits.hits.2.fields.field.0: -3.5 } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('scaled_float').get(1, 0.0)" + - match: { hits.hits.0.fields.field.0: 0.0 } + - match: { hits.hits.1.fields.field.0: 0.0 } + - match: { hits.hits.2.fields.field.0: 2.5 } + - do: search: rest_total_hits_as_int: true diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 7589d196fd5a1..ac3511bb24e11 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -42,6 +42,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.script.field.DocValuesField; +import org.elasticsearch.script.field.ScaledFloatDocValuesField; import org.elasticsearch.script.field.ToScriptField; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -270,7 +271,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new ScaledFloatIndexFieldData( scaledValues, scalingFactor, - (dv, n) -> new DelegateDocValuesField(new Doubles(new DoublesSupplier(dv)), n) + ScaledFloatDocValuesField::new ); }; } diff --git a/server/src/main/java/org/elasticsearch/script/field/ScaledFloatDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/ScaledFloatDocValuesField.java new file mode 100644 index 0000000000000..bdd5e9e32fa35 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/ScaledFloatDocValuesField.java @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.PrimitiveIterator; + +public class ScaledFloatDocValuesField implements DocValuesField, ScriptDocValues.Supplier { + + protected final SortedNumericDoubleValues input; + protected final String name; + + protected double[] values = new double[0]; + protected int count; + + private ScriptDocValues.Doubles doubles = null; + + public ScaledFloatDocValuesField(SortedNumericDoubleValues input, String name) { + this.input = input; + this.name = name; + } + + @Override + public void setNextDocId(int docId) throws IOException { + if (input.advanceExact(docId)) { + resize(input.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = input.nextValue(); + } + } else { + resize(0); + } + } + + protected void resize(int newSize) { + count = newSize; + + assert count >= 0 : "size must be positive (got " + count + "): likely integer overflow?"; + values = ArrayUtil.grow(values, count); + } + + @Override + public ScriptDocValues getScriptDocValues() { + if (doubles == null) { + doubles = new ScriptDocValues.Doubles(this); + } + + return doubles; + } + + @Override + public Double getInternal(int index) { + return values[index]; + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isEmpty() { + return count == 0; + } + + @Override + public int size() { + return count; + } + + public double get(double defaultValue) { + return get(0, defaultValue); + } + + public double get(int index, double defaultValue) { + if (isEmpty() || index < 0 || index >= count) { + return defaultValue; + } + + return values[index]; + } + + @Override + public PrimitiveIterator.OfDouble iterator() { + return new PrimitiveIterator.OfDouble() { + private int index = 0; + + @Override + public boolean hasNext() { + return index < count; + } + + @Override + public Double next() { + return nextDouble(); + } + + @Override + public double nextDouble() { + if (hasNext() == false) { + throw new NoSuchElementException(); + } + + return values[index++]; + } + }; + } +} From 056d6b2cb6b2e497c5a76be74709f7a8d27c9589 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 5 Jan 2022 10:36:18 -0800 Subject: [PATCH 2/4] Update docs/changelog/82275.yaml --- docs/changelog/82275.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/82275.yaml diff --git a/docs/changelog/82275.yaml b/docs/changelog/82275.yaml new file mode 100644 index 0000000000000..eab9560ad95c7 --- /dev/null +++ b/docs/changelog/82275.yaml @@ -0,0 +1,5 @@ +pr: 82275 +summary: Add scaled float to the scripting fields API +area: Infra/Scripting +type: enhancement +issues: [] From 1fefaa9824c12742a4c0e15bb32dc3909d3a99f6 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 5 Jan 2022 10:49:05 -0800 Subject: [PATCH 3/4] style fixes --- .../index/mapper/extras/ScaledFloatFieldMapper.java | 9 +-------- .../script/field/ScaledFloatDocValuesField.java | 1 - 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index ac3511bb24e11..2c8925ea477fe 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -25,8 +25,6 @@ import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.NumericDoubleValues; -import org.elasticsearch.index.fielddata.ScriptDocValues.Doubles; -import org.elasticsearch.index.fielddata.ScriptDocValues.DoublesSupplier; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -40,7 +38,6 @@ import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.script.field.DocValuesField; import org.elasticsearch.script.field.ScaledFloatDocValuesField; import org.elasticsearch.script.field.ToScriptField; @@ -268,11 +265,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S IndexNumericFieldData.NumericType.LONG, (dv, n) -> { throw new UnsupportedOperationException(); } ).build(cache, breakerService); - return new ScaledFloatIndexFieldData( - scaledValues, - scalingFactor, - ScaledFloatDocValuesField::new - ); + return new ScaledFloatIndexFieldData(scaledValues, scalingFactor, ScaledFloatDocValuesField::new); }; } diff --git a/server/src/main/java/org/elasticsearch/script/field/ScaledFloatDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/ScaledFloatDocValuesField.java index bdd5e9e32fa35..cecc8301bfc92 100644 --- a/server/src/main/java/org/elasticsearch/script/field/ScaledFloatDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/ScaledFloatDocValuesField.java @@ -13,7 +13,6 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import java.io.IOException; -import java.util.Iterator; import java.util.NoSuchElementException; import java.util.PrimitiveIterator; From feaba0560722a3bceea4704ec91cd82f25139c7b Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 5 Jan 2022 12:00:31 -0800 Subject: [PATCH 4/4] add unit test --- .../fielddata/FloatDocValuesFieldTests.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FloatDocValuesFieldTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FloatDocValuesFieldTests.java index 27cd9d08df132..c5f3b40975d36 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/FloatDocValuesFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/FloatDocValuesFieldTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.script.field.DoubleDocValuesField; import org.elasticsearch.script.field.FloatDocValuesField; +import org.elasticsearch.script.field.ScaledFloatDocValuesField; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -58,6 +59,27 @@ public void testDoubleField() throws IOException { } } + public void testScaledFloatField() throws IOException { + double[][] values = generate(ESTestCase::randomDouble); + ScaledFloatDocValuesField scaledFloatField = new ScaledFloatDocValuesField(wrap(values), "test"); + for (int round = 0; round < 10; round++) { + int d = between(0, values.length - 1); + scaledFloatField.setNextDocId(d); + if (values[d].length > 0) { + assertEquals(values[d][0], scaledFloatField.get(Double.MIN_VALUE), 0.0); + assertEquals(values[d][0], scaledFloatField.get(0, Double.MIN_VALUE), 0.0); + } + assertEquals(values[d].length, scaledFloatField.size()); + for (int i = 0; i < values[d].length; i++) { + assertEquals(values[d][i], scaledFloatField.get(i, Double.MIN_VALUE), 0.0); + } + int i = 0; + for (double dbl : scaledFloatField) { + assertEquals(values[d][i++], dbl, 0.0); + } + } + } + protected double[][] generate(DoubleSupplier supplier) { double[][] values = new double[between(3, 10)][]; for (int d = 0; d < values.length; d++) {