Skip to content

Commit 1469e18

Browse files
javannajavanna
authored
Add support for script parameter to boolean field mapper (#71454)
Relates to #68984
1 parent f944078 commit 1469e18

File tree

36 files changed

+630
-226
lines changed

36 files changed

+630
-226
lines changed

‎docs/reference/mapping/types/boolean.asciidoc‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,27 @@ The following parameters are accepted by `boolean` fields:
180180

181181
Accepts any of the true or false values listed above. The value is
182182
substituted for any explicit `null` values. Defaults to `null`, which
183-
means the field is treated as missing.
183+
means the field is treated as missing. Note that this cannot be set
184+
if the `script` parameter is used.
185+
186+
`on_script_error`::
187+
188+
Defines what to do if the script defined by the `script` parameter
189+
throws an error at indexing time. Accepts `reject` (default), which
190+
will cause the entire document to be rejected, and `ignore`, which
191+
will register the field in the document's
192+
<<mapping-ignored-field,`_ignored`>> metadata field and continue
193+
indexing. This parameter can only be set if the `script` field is
194+
also set.
195+
196+
`script`::
197+
198+
If this parameter is set, then the field will index values generated
199+
by this script, rather than reading the values directly from the
200+
source. If a value is set for this field on the input document, then
201+
the document will be rejected with an error.
202+
Scripts are in the same format as their
203+
<<runtime-mapping-fields,runtime equivalent>>.
184204

185205
<<mapping-store,`store`>>::
186206

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -545,7 +545,7 @@ public SearchAsYouTypeFieldMapper(String simpleName,
545545
PrefixFieldMapper prefixField,
546546
ShingleFieldMapper[] shingleFields,
547547
Builder builder) {
548-
super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo);
548+
super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo, false, null);
549549
this.prefixField = prefixField;
550550
this.shingleFields = shingleFields;
551551
this.maxShingleSize = builder.maxShingleSize.getValue();

‎modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
---
2+
setup:
3+
- do:
4+
indices.create:
5+
index: sensor
6+
body:
7+
settings:
8+
number_of_shards: 1
9+
number_of_replicas: 0
10+
mappings:
11+
properties:
12+
timestamp:
13+
type: date
14+
temperature:
15+
type: long
16+
voltage:
17+
type: double
18+
node:
19+
type: keyword
20+
over_v:
21+
type: boolean
22+
script:
23+
source: |
24+
for (def v : doc['voltage']) {
25+
emit(v >= params.min_v);
26+
}
27+
params:
28+
min_v: 5.0
29+
# Test fetching from _source
30+
over_v_from_source:
31+
type: boolean
32+
script:
33+
source: |
34+
emit(params._source.voltage >= 5.0);
35+
# Test many booleans
36+
big_vals:
37+
type: boolean
38+
script:
39+
source: |
40+
for (def v : doc['temperature']) {
41+
emit(v >= 200);
42+
}
43+
for (def v : doc['voltage']) {
44+
emit(v >= 5.0);
45+
}
46+
47+
48+
- do:
49+
bulk:
50+
index: sensor
51+
refresh: true
52+
body: |
53+
{"index":{}}
54+
{"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"}
55+
{"index":{}}
56+
{"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"}
57+
{"index":{}}
58+
{"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"}
59+
{"index":{}}
60+
{"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"}
61+
{"index":{}}
62+
{"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
63+
{"index":{}}
64+
{"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
65+
66+
---
67+
"get mapping":
68+
- do:
69+
indices.get_mapping:
70+
index: sensor
71+
- match: {sensor.mappings.properties.over_v.type: boolean }
72+
- match:
73+
sensor.mappings.properties.over_v.script.source: |
74+
for (def v : doc['voltage']) {
75+
emit(v >= params.min_v);
76+
}
77+
- match: {sensor.mappings.properties.over_v.script.params: {min_v: 5.0} }
78+
- match: {sensor.mappings.properties.over_v.script.lang: painless }
79+
80+
---
81+
"fetch fields":
82+
- do:
83+
search:
84+
index: sensor
85+
body:
86+
sort: timestamp
87+
fields: [over_v, over_v_from_source, big_vals]
88+
- match: {hits.total.value: 6}
89+
- match: {hits.hits.0.fields.over_v: [false] }
90+
- match: {hits.hits.0.fields.over_v_from_source: [false] }
91+
- match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues
92+
93+
---
94+
"docvalue_fields":
95+
- do:
96+
search:
97+
index: sensor
98+
body:
99+
sort: timestamp
100+
docvalue_fields: [over_v, over_v_from_source, big_vals]
101+
- match: {hits.total.value: 6}
102+
- match: {hits.hits.0.fields.over_v: [false] }
103+
- match: {hits.hits.0.fields.over_v_from_source: [false] }
104+
- match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues
105+
106+
---
107+
"terms agg":
108+
- do:
109+
search:
110+
index: sensor
111+
body:
112+
aggs:
113+
over_v:
114+
terms:
115+
field: over_v
116+
- match: {hits.total.value: 6}
117+
- match: {aggregations.over_v.buckets.0.key_as_string: "true"}
118+
- match: {aggregations.over_v.buckets.0.doc_count: 4}
119+
- match: {aggregations.over_v.buckets.1.key_as_string: "false"}
120+
- match: {aggregations.over_v.buckets.1.doc_count: 2}
121+
122+
---
123+
"term query":
124+
- do:
125+
search:
126+
index: sensor
127+
body:
128+
query:
129+
term:
130+
over_v: true
131+
sort:
132+
timestamp: asc
133+
- match: {hits.total.value: 4}
134+
- match: {hits.hits.0._source.voltage: 5.6}

‎server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedF
142142
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> ignoreZValue,
143143
MultiFields multiFields, CopyTo copyTo,
144144
Indexer<Parsed, Processed> indexer, Parser<Parsed> parser) {
145-
super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo);
145+
super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo, false, null);
146146
this.ignoreMalformed = ignoreMalformed;
147147
this.ignoreZValue = ignoreZValue;
148148
this.indexer = indexer;

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java‎

Lines changed: 59 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.apache.lucene.document.SortedNumericDocValuesField;
1414
import org.apache.lucene.document.StoredField;
1515
import org.apache.lucene.index.IndexOptions;
16+
import org.apache.lucene.index.LeafReaderContext;
1617
import org.apache.lucene.search.Query;
1718
import org.apache.lucene.search.TermRangeQuery;
1819
import org.apache.lucene.util.BytesRef;
@@ -25,14 +26,19 @@
2526
import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
2627
import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
2728
import org.elasticsearch.index.query.SearchExecutionContext;
29+
import org.elasticsearch.script.BooleanFieldScript;
30+
import org.elasticsearch.script.Script;
31+
import org.elasticsearch.script.ScriptCompiler;
2832
import org.elasticsearch.search.DocValueFormat;
33+
import org.elasticsearch.search.lookup.FieldValues;
2934
import org.elasticsearch.search.lookup.SearchLookup;
3035

3136
import java.io.IOException;
3237
import java.time.ZoneId;
3338
import java.util.Collections;
3439
import java.util.List;
3540
import java.util.Map;
41+
import java.util.Objects;
3642
import java.util.function.Supplier;
3743

3844
/**
@@ -72,43 +78,69 @@ public static class Builder extends FieldMapper.Builder {
7278
(n, c, o) -> o == null ? null : XContentMapValues.nodeBooleanValue(o), m -> toType(m).nullValue)
7379
.acceptsNull();
7480

81+
private final Parameter<Script> script = Parameter.scriptParam(m -> toType(m).script);
82+
private final Parameter<String> onScriptError = Parameter.onScriptErrorParam(m -> toType(m).onScriptError, script);
83+
7584
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
7685

77-
public Builder(String name) {
86+
private final ScriptCompiler scriptCompiler;
87+
88+
public Builder(String name, ScriptCompiler scriptCompiler) {
7889
super(name);
90+
this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
91+
this.script.precludesParameters(nullValue);
92+
this.script.setValidator(s -> {
93+
if (s != null && indexed.get() == false && docValues.get() == false) {
94+
throw new MapperParsingException("Cannot define script on field with index:false and doc_values:false");
95+
}
96+
});
7997
}
8098

8199
@Override
82100
protected List<Parameter<?>> getParameters() {
83-
return List.of(meta, docValues, indexed, nullValue, stored);
101+
return List.of(meta, docValues, indexed, nullValue, stored, script, onScriptError);
84102
}
85103

86104
@Override
87105
public BooleanFieldMapper build(ContentPath contentPath) {
88106
MappedFieldType ft = new BooleanFieldType(buildFullName(contentPath), indexed.getValue(), stored.getValue(),
89-
docValues.getValue(), nullValue.getValue(), meta.getValue());
107+
docValues.getValue(), nullValue.getValue(), scriptValues(), meta.getValue());
108+
90109
return new BooleanFieldMapper(name, ft, multiFieldsBuilder.build(this, contentPath), copyTo.build(), this);
91110
}
111+
112+
private FieldValues<Boolean> scriptValues() {
113+
if (script.get() == null) {
114+
return null;
115+
}
116+
BooleanFieldScript.Factory scriptFactory = scriptCompiler.compile(script.get(), BooleanFieldScript.CONTEXT);
117+
return scriptFactory == null ? null : (lookup, ctx, doc, consumer) -> scriptFactory
118+
.newFactory(name, script.get().getParams(), lookup)
119+
.newInstance(ctx)
120+
.runForDoc(doc, consumer);
121+
}
92122
}
93123

94-
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
124+
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.scriptCompiler()));
95125

96126
public static final class BooleanFieldType extends TermBasedFieldType {
97127

98128
private final Boolean nullValue;
129+
private final FieldValues<Boolean> scriptValues;
99130

100131
public BooleanFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues,
101-
Boolean nullValue, Map<String, String> meta) {
132+
Boolean nullValue, FieldValues<Boolean> scriptValues, Map<String, String> meta) {
102133
super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
103134
this.nullValue = nullValue;
135+
this.scriptValues = scriptValues;
104136
}
105137

106138
public BooleanFieldType(String name) {
107-
this(name, true, false, true, false, Collections.emptyMap());
139+
this(name, true, false, true, false, null, Collections.emptyMap());
108140
}
109141

110142
public BooleanFieldType(String name, boolean searchable) {
111-
this(name, searchable, false, true, false, Collections.emptyMap());
143+
this(name, searchable, false, true, false, null, Collections.emptyMap());
112144
}
113145

114146
@Override
@@ -121,7 +153,9 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
121153
if (format != null) {
122154
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
123155
}
124-
156+
if (this.scriptValues != null) {
157+
return FieldValues.valueFetcher(this.scriptValues, context);
158+
}
125159
return new SourceValueFetcher(name(), context, nullValue) {
126160
@Override
127161
protected Boolean parseSourceValue(Object value) {
@@ -208,14 +242,21 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
208242
private final boolean indexed;
209243
private final boolean hasDocValues;
210244
private final boolean stored;
245+
private final Script script;
246+
private final FieldValues<Boolean> scriptValues;
247+
private final ScriptCompiler scriptCompiler;
211248

212249
protected BooleanFieldMapper(String simpleName, MappedFieldType mappedFieldType,
213250
MultiFields multiFields, CopyTo copyTo, Builder builder) {
214-
super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo);
251+
super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo,
252+
builder.script.get() != null, builder.onScriptError.getValue());
215253
this.nullValue = builder.nullValue.getValue();
216254
this.stored = builder.stored.getValue();
217255
this.indexed = builder.indexed.getValue();
218256
this.hasDocValues = builder.docValues.getValue();
257+
this.script = builder.script.get();
258+
this.scriptValues = builder.scriptValues();
259+
this.scriptCompiler = builder.scriptCompiler;
219260
}
220261

221262
@Override
@@ -240,7 +281,10 @@ protected void parseCreateField(ParseContext context) throws IOException {
240281
value = context.parser().booleanValue();
241282
}
242283
}
284+
indexValue(context, value);
285+
}
243286

287+
private void indexValue(ParseContext context, Boolean value) {
244288
if (value == null) {
245289
return;
246290
}
@@ -257,14 +301,18 @@ protected void parseCreateField(ParseContext context) throws IOException {
257301
}
258302
}
259303

304+
@Override
305+
protected void indexScriptValues(SearchLookup searchLookup, LeafReaderContext readerContext, int doc, ParseContext parseContext) {
306+
this.scriptValues.valuesForDoc(searchLookup, readerContext, doc, value -> indexValue(parseContext, value));
307+
}
308+
260309
@Override
261310
public FieldMapper.Builder getMergeBuilder() {
262-
return new Builder(simpleName()).init(this);
311+
return new Builder(simpleName(), scriptCompiler).init(this);
263312
}
264313

265314
@Override
266315
protected String contentType() {
267316
return CONTENT_TYPE;
268317
}
269-
270318
}

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.common.time.DateFormatter;
1616
import org.elasticsearch.common.xcontent.XContentParser;
1717
import org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
18+
import org.elasticsearch.script.ScriptCompiler;
1819

1920
import java.io.IOException;
2021
import java.time.format.DateTimeParseException;
@@ -274,7 +275,7 @@ public void newDynamicLongField(ParseContext context, String name) throws IOExce
274275
new NumberFieldMapper.Builder(
275276
name,
276277
NumberFieldMapper.NumberType.LONG,
277-
null,
278+
ScriptCompiler.NONE,
278279
context.indexSettings().getSettings()
279280
), context);
280281
}
@@ -287,13 +288,13 @@ public void newDynamicDoubleField(ParseContext context, String name) throws IOEx
287288
createDynamicField(new NumberFieldMapper.Builder(
288289
name,
289290
NumberFieldMapper.NumberType.FLOAT,
290-
null,
291+
ScriptCompiler.NONE,
291292
context.indexSettings().getSettings()), context);
292293
}
293294

294295
@Override
295296
public void newDynamicBooleanField(ParseContext context, String name) throws IOException {
296-
createDynamicField(new BooleanFieldMapper.Builder(name), context);
297+
createDynamicField(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE), context);
297298
}
298299

299300
@Override

0 commit comments

Comments
 (0)