Skip to content

Commit 5ff1e75

Browse files
nik9000nik9000
committed
Add long flavored script field
This adds the `long` runtime type to `script` fields and implements doc values, field data, the `term` and `exists` query.
1 parent 4ba86b1 commit 5ff1e75

26 files changed

+1195
-185
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -701,23 +701,7 @@ public List<Field> createFields(String name, Number value,
701701
LONG("long", NumericType.LONG) {
702702
@Override
703703
public Long parse(Object value, boolean coerce) {
704-
if (value instanceof Long) {
705-
return (Long)value;
706-
}
707-
708-
double doubleValue = objectToDouble(value);
709-
// this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will
710-
// be equal to Long.MAX_VALUE after conversion to double. More checks ahead.
711-
if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
712-
throw new IllegalArgumentException("Value [" + value + "] is out of range for a long");
713-
}
714-
if (!coerce && doubleValue % 1 != 0) {
715-
throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
716-
}
717-
718-
// longs need special handling so we don't lose precision while parsing
719-
String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString();
720-
return Numbers.toLong(stringValue, coerce);
704+
return objectToLong(value, coerce);
721705
}
722706

723707
@Override
@@ -854,7 +838,7 @@ Number valueForSearch(Number value) {
854838
/**
855839
* Returns true if the object is a number and has a decimal part
856840
*/
857-
boolean hasDecimalPart(Object number) {
841+
public static boolean hasDecimalPart(Object number) {
858842
if (number instanceof Number) {
859843
double doubleValue = ((Number) number).doubleValue();
860844
return doubleValue % 1 != 0;
@@ -898,6 +882,30 @@ private static double objectToDouble(Object value) {
898882

899883
return doubleValue;
900884
}
885+
886+
/**
887+
* Converts and Object to a {@code long} by checking it against known
888+
* types and checking its range.
889+
*/
890+
public static long objectToLong(Object value, boolean coerce) {
891+
if (value instanceof Long) {
892+
return (Long)value;
893+
}
894+
895+
double doubleValue = objectToDouble(value);
896+
// this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will
897+
// be equal to Long.MAX_VALUE after conversion to double. More checks ahead.
898+
if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
899+
throw new IllegalArgumentException("Value [" + value + "] is out of range for a long");
900+
}
901+
if (!coerce && doubleValue % 1 != 0) {
902+
throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
903+
}
904+
905+
// longs need special handling so we don't lose precision while parsing
906+
String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString();
907+
return Numbers.toLong(stringValue, coerce);
908+
}
901909
}
902910

903911
public static final class NumberFieldType extends SimpleMappedFieldType {

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java‎

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
package org.elasticsearch.xpack.runtimefields;
88

99
import org.apache.lucene.index.LeafReaderContext;
10+
import org.apache.lucene.util.ArrayUtil;
1011
import org.elasticsearch.painless.spi.Whitelist;
1112
import org.elasticsearch.painless.spi.WhitelistLoader;
1213
import org.elasticsearch.script.ScriptContext;
@@ -16,10 +17,9 @@
1617
import java.io.IOException;
1718
import java.util.List;
1819
import java.util.Map;
19-
import java.util.function.LongConsumer;
2020

2121
public abstract class LongScriptFieldScript extends AbstractScriptFieldScript {
22-
static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("long_script_field", Factory.class);
22+
public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("long_script_field", Factory.class);
2323

2424
static List<Whitelist> whitelist() {
2525
return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt"));
@@ -32,14 +32,47 @@ public interface Factory extends ScriptFactory {
3232
}
3333

3434
public interface LeafFactory {
35-
LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException;
35+
LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
3636
}
3737

38-
private final LongConsumer sync;
38+
private long[] values = new long[1];
39+
private int count;
3940

40-
public LongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx, LongConsumer sync) {
41+
public LongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
4142
super(params, searchLookup, ctx);
42-
this.sync = sync;
43+
}
44+
45+
/**
46+
* Execute the script for the provided {@code docId}.
47+
*/
48+
public final void runForDoc(int docId) {
49+
count = 0;
50+
setDocument(docId);
51+
execute();
52+
}
53+
54+
/**
55+
* Values from the last time {@link #runForDoc(int)} was called. This array
56+
* is mutable and will change with the next call of {@link #runForDoc(int)}.
57+
* It is also oversized and will contain garbage at all indices at and
58+
* above {@link #count()}.
59+
*/
60+
public final long[] values() {
61+
return values;
62+
}
63+
64+
/**
65+
* The number of results produced the last time {@link #runForDoc(int)} was called.
66+
*/
67+
public final int count() {
68+
return count;
69+
}
70+
71+
protected final void add(long v) {
72+
if (values.length < count + 1) {
73+
values = ArrayUtil.grow(values, count + 1);
74+
}
75+
values[count++] = v;
4376
}
4477

4578
public static class Value {
@@ -50,7 +83,7 @@ public Value(LongScriptFieldScript script) {
5083
}
5184

5285
public void value(long v) {
53-
script.sync.accept(v);
86+
script.add(v);
5487
}
5588
}
5689
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ private ScriptBinaryFieldData(IndexSettings indexSettings, String fieldName, Str
7272
this.scriptFactory = scriptFactory;
7373
}
7474

75+
@Override
7576
public void setSearchLookup(SearchLookup searchLookup) {
7677
// TODO wire the params from the mappings definition, we don't parse them yet
7778
this.leafFactory.set(scriptFactory.newFactory(Collections.emptyMap(), searchLookup));

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.runtimefields.fielddata;
8+
9+
import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues;
10+
import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript;
11+
12+
import java.io.IOException;
13+
import java.util.Arrays;
14+
15+
public final class ScriptLongDocValues extends AbstractSortedNumericDocValues {
16+
private final LongScriptFieldScript script;
17+
private int cursor;
18+
19+
ScriptLongDocValues(LongScriptFieldScript script) {
20+
this.script = script;
21+
}
22+
23+
@Override
24+
public boolean advanceExact(int docId) {
25+
script.runForDoc(docId);
26+
if (script.count() == 0) {
27+
return false;
28+
}
29+
Arrays.sort(script.values(), 0, script.count());
30+
cursor = 0;
31+
return true;
32+
}
33+
34+
@Override
35+
public long nextValue() throws IOException {
36+
return script.values()[cursor++];
37+
}
38+
39+
@Override
40+
public int docValueCount() {
41+
return script.count();
42+
}
43+
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongFieldData.java‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.runtimefields.fielddata;
8+
9+
import org.apache.lucene.index.LeafReaderContext;
10+
import org.apache.lucene.index.SortedNumericDocValues;
11+
import org.apache.lucene.util.SetOnce;
12+
import org.elasticsearch.ElasticsearchException;
13+
import org.elasticsearch.index.Index;
14+
import org.elasticsearch.index.IndexSettings;
15+
import org.elasticsearch.index.fielddata.FieldData;
16+
import org.elasticsearch.index.fielddata.IndexFieldData;
17+
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
18+
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
19+
import org.elasticsearch.index.fielddata.LeafNumericFieldData;
20+
import org.elasticsearch.index.fielddata.ScriptDocValues;
21+
import org.elasticsearch.index.fielddata.SearchLookupAware;
22+
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
23+
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
24+
import org.elasticsearch.index.mapper.MappedFieldType;
25+
import org.elasticsearch.index.mapper.MapperService;
26+
import org.elasticsearch.indices.breaker.CircuitBreakerService;
27+
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
28+
import org.elasticsearch.search.aggregations.support.ValuesSourceType;
29+
import org.elasticsearch.search.lookup.SearchLookup;
30+
import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript;
31+
32+
import java.io.IOException;
33+
import java.util.ArrayList;
34+
import java.util.Collections;
35+
import java.util.List;
36+
37+
public final class ScriptLongFieldData extends IndexNumericFieldData implements SearchLookupAware {
38+
39+
public static class Builder implements IndexFieldData.Builder {
40+
41+
private final LongScriptFieldScript.Factory scriptFactory;
42+
43+
public Builder(LongScriptFieldScript.Factory scriptFactory) {
44+
this.scriptFactory = scriptFactory;
45+
}
46+
47+
@Override
48+
public ScriptLongFieldData build(
49+
IndexSettings indexSettings,
50+
MappedFieldType fieldType,
51+
IndexFieldDataCache cache,
52+
CircuitBreakerService breakerService,
53+
MapperService mapperService
54+
) {
55+
return new ScriptLongFieldData(indexSettings.getIndex(), fieldType.name(), scriptFactory);
56+
}
57+
}
58+
59+
private final Index index;
60+
private final String fieldName;
61+
private final LongScriptFieldScript.Factory scriptFactory;
62+
private final SetOnce<LongScriptFieldScript.LeafFactory> leafFactory = new SetOnce<>();
63+
64+
private ScriptLongFieldData(Index index, String fieldName, LongScriptFieldScript.Factory scriptFactory) {
65+
this.index = index;
66+
this.fieldName = fieldName;
67+
this.scriptFactory = scriptFactory;
68+
}
69+
70+
@Override
71+
public void setSearchLookup(SearchLookup searchLookup) {
72+
// TODO wire the params from the mappings definition, we don't parse them yet
73+
this.leafFactory.set(scriptFactory.newFactory(Collections.emptyMap(), searchLookup));
74+
}
75+
76+
@Override
77+
public String getFieldName() {
78+
return fieldName;
79+
}
80+
81+
@Override
82+
public ValuesSourceType getValuesSourceType() {
83+
return CoreValuesSourceType.NUMERIC;
84+
}
85+
86+
@Override
87+
public ScriptLongLeafFieldData load(LeafReaderContext context) {
88+
try {
89+
return loadDirect(context);
90+
} catch (Exception e) {
91+
if (e instanceof ElasticsearchException) {
92+
throw (ElasticsearchException) e;
93+
} else {
94+
throw new ElasticsearchException(e);
95+
}
96+
}
97+
}
98+
99+
@Override
100+
public ScriptLongLeafFieldData loadDirect(LeafReaderContext context) throws IOException {
101+
return new ScriptLongLeafFieldData(new ScriptLongDocValues(leafFactory.get().newInstance(context)));
102+
}
103+
104+
@Override
105+
public NumericType getNumericType() {
106+
return NumericType.LONG;
107+
}
108+
109+
@Override
110+
protected boolean sortRequiresCustomComparator() {
111+
return true;
112+
}
113+
114+
@Override
115+
public void clear() {}
116+
117+
@Override
118+
public Index index() {
119+
return index;
120+
}
121+
122+
public static class ScriptLongLeafFieldData implements LeafNumericFieldData {
123+
private final ScriptLongDocValues scriptBinaryDocValues;
124+
125+
ScriptLongLeafFieldData(ScriptLongDocValues scriptBinaryDocValues) {
126+
this.scriptBinaryDocValues = scriptBinaryDocValues;
127+
}
128+
129+
@Override
130+
public ScriptDocValues<?> getScriptValues() {
131+
return new ScriptDocValues.Longs(getLongValues());
132+
}
133+
134+
@Override
135+
public SortedBinaryDocValues getBytesValues() {
136+
return FieldData.toString(scriptBinaryDocValues);
137+
}
138+
139+
@Override
140+
public SortedNumericDoubleValues getDoubleValues() {
141+
return FieldData.castToDouble(getLongValues());
142+
}
143+
144+
@Override
145+
public SortedNumericDocValues getLongValues() {
146+
return scriptBinaryDocValues;
147+
}
148+
149+
@Override
150+
public long ramBytesUsed() {
151+
return 0;
152+
}
153+
154+
@Override
155+
public void close() {
156+
157+
}
158+
}
159+
160+
static class ScriptBinaryResult {
161+
private final List<String> result = new ArrayList<>();
162+
163+
void accept(String value) {
164+
this.result.add(value);
165+
}
166+
167+
List<String> getResult() {
168+
return result;
169+
}
170+
}
171+
}

0 commit comments

Comments
 (0)