diff --git a/docs/changelog/81476.yaml b/docs/changelog/81476.yaml new file mode 100644 index 0000000000000..cbc91db476c8e --- /dev/null +++ b/docs/changelog/81476.yaml @@ -0,0 +1,5 @@ +pr: 81476 +summary: "Script: fields API for x-pack version, doc version, seq no, mumur3" +area: Infra/Scripting +type: enhancement +issues: [] diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt index 71c01ad7eddcb..27934c7eacfb9 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt @@ -95,3 +95,16 @@ class org.elasticsearch.script.field.IpDocValuesField @dynamic_type { String asString(String) String asString(int, String) } + +class org.elasticsearch.script.field.AbstractLongDocValuesField @dynamic_type { + long get(long) + long get(int, long) +} + +# subclass of AbstractLongDocValuesField +class org.elasticsearch.script.field.SeqNoDocValuesField @dynamic_type { +} + +# subclass of AbstractLongDocValuesField +class org.elasticsearch.script.field.VersionDocValuesField @dynamic_type { +} diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 7adb2c87e6541..898c60c60d878 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -1406,3 +1406,81 @@ setup: script: source: "int value = field('dne').get(1, 1); value" - match: { hits.hits.0.fields.field.0: 1 } + +--- +"version and sequence number": + - do: + indices.create: + index: versiontest + body: + settings: + number_of_shards: 1 + mappings: + properties: + keyword: + type: keyword + + - do: + index: + index: test + id: 3000 + version: 50 + version_type: external + body: + keyword: "3k" + + - do: + indices.refresh: {} + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: 3000 } } + script_fields: + ver: + script: + source: "field('_version').get(10000)" + seq: + script: + source: "field('_seq_no').get(10000)" + - match: { hits.hits.0.fields.ver.0: 50 } + - match: { hits.hits.0.fields.seq.0: 3 } + + - do: + index: + index: test + id: 3000 + version: 60 + version_type: external + body: + keyword: "3k+1" + - do: + indices.refresh: {} + + - do: + catch: conflict + index: + index: test + id: 3000 + version: 55 + version_type: external + body: + keyword: "3k+2" + - do: + indices.refresh: {} + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: 3000 } } + script_fields: + ver: + script: + source: "field('_version').get(10000)" + seq: + script: + source: "field('_seq_no').get(10000)" + - match: { hits.hits.0.fields.ver.0: 60 } + - match: { hits.hits.0.fields.seq.0: 4 } diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle index 81df70e3855bf..e6a4183139cbf 100644 --- a/plugins/mapper-murmur3/build.gradle +++ b/plugins/mapper-murmur3/build.gradle @@ -11,6 +11,12 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.' classname 'org.elasticsearch.plugin.mapper.MapperMurmur3Plugin' + extendedPlugins = ['lang-painless'] +} + +dependencies { + compileOnly project(':modules:lang-painless:spi') + testImplementation project(':modules:lang-painless') } restResources { @@ -18,3 +24,8 @@ restResources { include '_common', 'indices', 'index', 'search' } } + +testClusters.configureEach { + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'false' +} diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index dafc303dae601..036a76e548668 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -12,14 +12,11 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; -import org.elasticsearch.index.fielddata.ScriptDocValues.Longs; -import org.elasticsearch.index.fielddata.ScriptDocValues.LongsSupplier; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; @@ -29,8 +26,7 @@ import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.script.field.DelegateDocValuesField; -import org.elasticsearch.script.field.ToScriptField; +import org.elasticsearch.script.field.murmur3.Murmur3DocValueField; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -84,11 +80,6 @@ public Murmur3FieldMapper build(MapperBuilderContext context) { // this only exists so a check can be done to match the field type to using murmur3 hashing... public static class Murmur3FieldType extends MappedFieldType { - public static final ToScriptField TO_SCRIPT_FIELD = (dv, n) -> new DelegateDocValuesField( - new Longs(new LongsSupplier(dv)), - n - ); - private Murmur3FieldType(String name, boolean isStored, Map meta) { super(name, false, isStored, true, TextSearchInfo.NONE, meta); } @@ -101,7 +92,7 @@ public String typeName() { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); - return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, TO_SCRIPT_FIELD); + return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, Murmur3DocValueField::new); } @Override diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3DocValueField.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3DocValueField.java new file mode 100644 index 0000000000000..2f880c782f1d8 --- /dev/null +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3DocValueField.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field.murmur3; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.script.field.AbstractLongDocValuesField; + +public class Murmur3DocValueField extends AbstractLongDocValuesField { + + public Murmur3DocValueField(SortedNumericDocValues input, String name) { + super(input, name); + } +} diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3PainlessExtension.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3PainlessExtension.java new file mode 100644 index 0000000000000..344642482947f --- /dev/null +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/script/field/murmur3/Murmur3PainlessExtension.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field.murmur3; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.script.ScriptModule.CORE_CONTEXTS; + +public class Murmur3PainlessExtension implements PainlessExtension { + + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles( + Murmur3PainlessExtension.class, + "org.elasticsearch.field.murmur3.txt" + ); + + @Override + public Map, List> getContextWhitelists() { + List whitelist = singletonList(WHITELIST); + Map, List> contextWhitelists = new HashMap<>(CORE_CONTEXTS.size()); + for (ScriptContext scriptContext : CORE_CONTEXTS.values()) { + contextWhitelists.put(scriptContext, whitelist); + } + return contextWhitelists; + } +} diff --git a/plugins/mapper-murmur3/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/plugins/mapper-murmur3/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..f5b8d5f84c6e1 --- /dev/null +++ b/plugins/mapper-murmur3/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.script.field.murmur3.Murmur3PainlessExtension diff --git a/plugins/mapper-murmur3/src/main/resources/org/elasticsearch/script/field/murmur3/org.elasticsearch.field.murmur3.txt b/plugins/mapper-murmur3/src/main/resources/org/elasticsearch/script/field/murmur3/org.elasticsearch.field.murmur3.txt new file mode 100644 index 0000000000000..ae866177cba6e --- /dev/null +++ b/plugins/mapper-murmur3/src/main/resources/org/elasticsearch/script/field/murmur3/org.elasticsearch.field.murmur3.txt @@ -0,0 +1,11 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +# subclass of AbstractLongDocValuesField +class org.elasticsearch.script.field.murmur3.Murmur3DocValueField @dynamic_type { +} diff --git a/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml b/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml index 34db9d017a854..3ed6e6a97c2c2 100644 --- a/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml +++ b/plugins/mapper-murmur3/src/yamlRestTest/resources/rest-api-spec/test/mapper_murmur3/10_basic.yml @@ -1,9 +1,7 @@ # Integration tests for Mapper Murmur3 components # ---- -"Mapper Murmur3": - +setup: - do: indices.create: index: test @@ -20,6 +18,8 @@ - do: indices.refresh: {} +--- +"Mapper Murmur3": - do: search: rest_total_hits_as_int: true @@ -60,3 +60,27 @@ body: { "aggs": { "foo_count": { "cardinality": { "field": "foo.hash" } } } } - match: { aggregations.foo_count.value: 3 } + +--- +"Mumur3 script fields api": + + - do: + index: + index: test + id: 1 + body: { "foo": "foo" } + + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + sort: [ { foo.hash: desc } ] + script_fields: + field: + script: + source: "field('foo.hash').get(10L)" + + - match: { hits.hits.0.fields.field.0: -2129773440516405919 } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 095f2ad6cbd0b..d42ca38fb82a1 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.fielddata; -import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -94,48 +93,6 @@ protected void throwIfEmpty() { } } - public static class LongsSupplier implements Supplier { - - private final SortedNumericDocValues in; - private long[] values = new long[0]; - private int count; - - public LongsSupplier(SortedNumericDocValues in) { - this.in = in; - } - - @Override - public void setNextDocId(int docId) throws IOException { - if (in.advanceExact(docId)) { - resize(in.docValueCount()); - for (int i = 0; i < count; i++) { - values[i] = in.nextValue(); - } - } else { - resize(0); - } - } - - /** - * Set the {@link #size()} and ensure that the {@link #values} array can - * store at least that many entries. - */ - private void resize(int newSize) { - count = newSize; - values = ArrayUtil.grow(values, count); - } - - @Override - public Long getInternal(int index) { - return values[index]; - } - - @Override - public int size() { - return count; - } - } - public static class Longs extends ScriptDocValues { public Longs(Supplier supplier) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index e21186ca678bd..d487a987b003e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -17,12 +17,10 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; -import org.elasticsearch.index.fielddata.ScriptDocValues.Longs; -import org.elasticsearch.index.fielddata.ScriptDocValues.LongsSupplier; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.seqno.SequenceNumbers; -import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.script.field.SeqNoDocValuesField; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -186,11 +184,7 @@ public Query rangeQuery( @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); - return new SortedNumericIndexFieldData.Builder( - name(), - NumericType.LONG, - (dv, n) -> new DelegateDocValuesField(new Longs(new LongsSupplier(dv)), n) - ); + return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, SeqNoDocValuesField::new); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index aa82c3371c508..706a862b989b5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -13,12 +13,10 @@ import org.apache.lucene.search.Query; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; -import org.elasticsearch.index.fielddata.ScriptDocValues.Longs; -import org.elasticsearch.index.fielddata.ScriptDocValues.LongsSupplier; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.script.field.VersionDocValuesField; import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; @@ -60,11 +58,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); - return new SortedNumericIndexFieldData.Builder( - name(), - NumericType.LONG, - (dv, n) -> new DelegateDocValuesField(new Longs(new LongsSupplier(dv)), n) - ); + return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, VersionDocValuesField::new); } } diff --git a/server/src/main/java/org/elasticsearch/script/field/AbstractLongDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/AbstractLongDocValuesField.java new file mode 100644 index 0000000000000..41d59ea15848f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/AbstractLongDocValuesField.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.index.fielddata.ScriptDocValues; + +import java.io.IOException; +import java.util.NoSuchElementException; +import java.util.PrimitiveIterator; + +public abstract class AbstractLongDocValuesField implements ScriptDocValues.Supplier, DocValuesField { + + protected final String name; + // used for backwards compatibility for old-style "doc" access + // as a delegate to this field class + protected ScriptDocValues scriptDocValues = null; + + protected final SortedNumericDocValues input; + protected long[] values = new long[0]; + protected int count; + + public AbstractLongDocValuesField(SortedNumericDocValues input, String name) { + this.input = input; + this.name = name; + } + + /** + * Override if not using {@link ScriptDocValues.Longs} + */ + protected ScriptDocValues newScriptDocValues() { + return new ScriptDocValues.Longs(this); + } + + /** + * Override if long has special formatting. + */ + protected long formatLong(long raw) { + return raw; + } + + @Override + public void setNextDocId(int docId) throws IOException { + if (input.advanceExact(docId)) { + resize(input.docValueCount()); + for (int i = 0; i < count; i++) { + values[i] = formatLong(input.nextValue()); + } + } else { + resize(0); + } + } + + @Override + public ScriptDocValues getScriptDocValues() { + if (scriptDocValues == null) { + scriptDocValues = newScriptDocValues(); + } + + return scriptDocValues; + } + + /** + * Set the {@link #size()} and ensure that the {@link #values} array can + * store at least that many entries. + */ + private void resize(int newSize) { + count = newSize; + values = ArrayUtil.grow(values, count); + } + + // this method is required to support the Long return values + // for the old-style "doc" access in ScriptDocValues + @Override + public Long getInternal(int index) { + return getLong(index); + } + + protected long getLong(int index) { + return values[index]; + } + + @Override + public int size() { + return count; + } + + @Override + public boolean isEmpty() { + return count == 0; + } + + @Override + public String getName() { + return name; + } + + @Override + public PrimitiveIterator.OfLong iterator() { + return new PrimitiveIterator.OfLong() { + private int index = 0; + + @Override + public boolean hasNext() { + return index < count; + } + + @Override + public Long next() { + return nextLong(); + } + + @Override + public long nextLong() { + if (hasNext() == false) { + throw new NoSuchElementException(); + } + + return getLong(index++); + } + }; + } + + /** Returns the 0th index value as an {@code long} if it exists, otherwise {@code defaultValue}. */ + public long get(long defaultValue) { + return get(0, defaultValue); + } + + /** Returns the value at {@code index} as an {@code long} if it exists, otherwise {@code defaultValue}. */ + public long get(int index, long defaultValue) { + if (isEmpty() || index < 0 || index >= count) { + return defaultValue; + } + + return getLong(index); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/LongDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/LongDocValuesField.java index 0ff7b9197a503..7f9ebb7f91e18 100644 --- a/server/src/main/java/org/elasticsearch/script/field/LongDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/LongDocValuesField.java @@ -9,119 +9,10 @@ package org.elasticsearch.script.field; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.util.ArrayUtil; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import java.io.IOException; -import java.util.Iterator; -import java.util.NoSuchElementException; - -public class LongDocValuesField implements DocValuesField, ScriptDocValues.Supplier { - - protected final SortedNumericDocValues input; - protected final String name; - - protected long[] values = new long[0]; - protected int count; - - private ScriptDocValues.Longs longs = null; +public class LongDocValuesField extends AbstractLongDocValuesField { public LongDocValuesField(SortedNumericDocValues input, String name) { - this.input = input; - this.name = name; - } - - @Override - public void setNextDocId(int docId) throws IOException { - if (input.advanceExact(docId)) { - resize(input.docValueCount()); - for (int i = 0; i < count; i++) { - values[i] = input.nextValue(); - } - } else { - resize(0); - } - } - - protected void resize(int newSize) { - count = newSize; - - assert count >= 0 : "size must be positive (got " + count + "): likely integer overflow?"; - values = ArrayUtil.grow(values, count); - } - - /** - * Returns a {@code ScriptDocValues} of the appropriate type for this field. - * This is used to support backwards compatibility for accessing field values - * through the {@code doc} variable. - */ - @Override - public ScriptDocValues getScriptDocValues() { - if (longs == null) { - longs = new ScriptDocValues.Longs(this); - } - - return longs; - } - - @Override - public Long getInternal(int index) { - return values[index]; - } - - /** - * Returns the name of this field. - */ - @Override - public String getName() { - return name; - } - - /** - * Returns {@code true} if this field has no values, otherwise {@code false}. - */ - @Override - public boolean isEmpty() { - return count == 0; - } - - /** - * Returns the number of values this field has. - */ - @Override - public int size() { - return count; - } - - @Override - public Iterator iterator() { - return new Iterator() { - private int index = 0; - - @Override - public boolean hasNext() { - return index < count; - } - - @Override - public Long next() { - if (hasNext() == false) { - throw new NoSuchElementException(); - } - return values[index++]; - } - }; - } - - public long get(long defaultValue) { - return get(0, defaultValue); - } - - public long get(int index, long defaultValue) { - if (isEmpty() || index < 0 || index >= count) { - return defaultValue; - } - - return values[index]; + super(input, name); } } diff --git a/server/src/main/java/org/elasticsearch/script/field/SeqNoDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/SeqNoDocValuesField.java new file mode 100644 index 0000000000000..86dcc1fa7d788 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/SeqNoDocValuesField.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.apache.lucene.index.SortedNumericDocValues; + +public class SeqNoDocValuesField extends AbstractLongDocValuesField { + public SeqNoDocValuesField(SortedNumericDocValues input, String name) { + super(input, name); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/VersionDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/VersionDocValuesField.java new file mode 100644 index 0000000000000..4a51b6ec8e303 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/VersionDocValuesField.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.apache.lucene.index.SortedNumericDocValues; + +public class VersionDocValuesField extends AbstractLongDocValuesField { + public VersionDocValuesField(SortedNumericDocValues input, String name) { + super(input, name); + } +} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java index 209a412e32ad8..b7c7cdcd67a4b 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java @@ -8,92 +8,35 @@ package org.elasticsearch.xpack.unsignedlong; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.script.field.DocValuesField; +import org.elasticsearch.script.field.AbstractLongDocValuesField; -import java.io.IOException; import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.NoSuchElementException; -import java.util.PrimitiveIterator; import static org.elasticsearch.search.DocValueFormat.MASK_2_63; import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.BIGINTEGER_2_64_MINUS_ONE; -public class UnsignedLongDocValuesField implements DocValuesField, ScriptDocValues.Supplier { - - private final SortedNumericDocValues input; - private final String name; - - private long[] values = new long[0]; - private int count = 0; - - // used for backwards compatibility for old-style "doc" access - // as a delegate to this field class - private UnsignedLongScriptDocValues unsignedLongScriptDocValues = null; +public class UnsignedLongDocValuesField extends AbstractLongDocValuesField { public UnsignedLongDocValuesField(SortedNumericDocValues input, String name) { - this.input = input; - this.name = name; - } - - @Override - public void setNextDocId(int docId) throws IOException { - if (input.advanceExact(docId)) { - resize(input.docValueCount()); - for (int i = 0; i < count; i++) { - values[i] = input.nextValue(); - } - } else { - resize(0); - } - } - - private void resize(int newSize) { - count = newSize; - values = ArrayUtil.grow(values, count); - } - - @Override - public ScriptDocValues getScriptDocValues() { - if (unsignedLongScriptDocValues == null) { - unsignedLongScriptDocValues = new UnsignedLongScriptDocValues(this); - } - - return unsignedLongScriptDocValues; - } - - @Override - public String getName() { - return name; - } - - @Override - public boolean isEmpty() { - return count == 0; + super(input, name); } - // this method is required to support the Long return values - // for the old-style "doc" access in ScriptDocValues @Override - public Long getInternal(int index) { - return toFormatted(index); - } - - @Override - public int size() { - return count; + public ScriptDocValues newScriptDocValues() { + return new UnsignedLongScriptDocValues(this); } /** * Applies the formatting from {@link org.elasticsearch.search.DocValueFormat.UnsignedLongShiftedDocValueFormat#format(long)} so * that the underlying value can be treated as a primitive long as that method returns either a {@code long} or a {@code BigInteger}. */ - protected long toFormatted(int index) { - return values[index] ^ MASK_2_63; + @Override + protected long formatLong(long raw) { + return raw ^ MASK_2_63; } /** Return all the values as a {@code List}. */ @@ -105,26 +48,12 @@ public List getValues() { List longValues = new ArrayList<>(count); for (int index = 0; index < count; ++index) { - longValues.add(toFormatted(index)); + longValues.add(getLong(index)); } return longValues; } - /** Returns the 0th index value as an {@code long} if it exists, otherwise {@code defaultValue}. */ - public long get(long defaultValue) { - return get(0, defaultValue); - } - - /** Returns the value at {@code index} as an {@code long} if it exists, otherwise {@code defaultValue}. */ - public long get(int index, long defaultValue) { - if (isEmpty() || index < 0 || index >= count) { - return defaultValue; - } - - return toFormatted(index); - } - /** Returns the 0th index value as an {@code long} if it exists, otherwise {@code defaultValue}. */ public long getValue(long defaultValue) { return get(0, defaultValue); @@ -135,34 +64,8 @@ public long getValue(int index, long defaultValue) { return get(index, defaultValue); } - @Override - public PrimitiveIterator.OfLong iterator() { - return new PrimitiveIterator.OfLong() { - private int index = 0; - - @Override - public boolean hasNext() { - return index < count; - } - - @Override - public Long next() { - return nextLong(); - } - - @Override - public long nextLong() { - if (hasNext() == false) { - throw new NoSuchElementException(); - } - - return toFormatted(index++); - } - }; - } - protected BigInteger toBigInteger(int index) { - return BigInteger.valueOf(toFormatted(index)).and(BIGINTEGER_2_64_MINUS_ONE); + return BigInteger.valueOf(getLong(index)).and(BIGINTEGER_2_64_MINUS_ONE); } /** Converts all the values to {@code BigInteger} and returns them as a {@code List}. */ diff --git a/x-pack/plugin/mapper-version/build.gradle b/x-pack/plugin/mapper-version/build.gradle index 557d43c0d9f18..054acbea8c16f 100644 --- a/x-pack/plugin/mapper-version/build.gradle +++ b/x-pack/plugin/mapper-version/build.gradle @@ -1,11 +1,14 @@ evaluationDependsOn(xpackModule('core')) + apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { name 'mapper-version' - description 'A plugin for a field type to store sofware versions' + description 'A plugin for a field type to store software versions' classname 'org.elasticsearch.xpack.versionfield.VersionFieldPlugin' extendedPlugins = ['x-pack-core', 'lang-painless'] } diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/Version.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/Version.java new file mode 100644 index 0000000000000..e0113fd7443ee --- /dev/null +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/Version.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Script value class. + * TODO(stu): implement {@code Comparable} based on {@code VersionEncoder#prefixDigitGroupsWithLength(String, BytesRefBuilder)} + * See: https://github.com/elastic/elasticsearch/issues/82287 + */ +public class Version implements ToXContent { + protected String version; + + public Version(String version) { + this.version = version; + } + + @Override + public String toString() { + return version; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.value(toString()); + } + + @Override + public boolean isFragment() { + return false; + } +} diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java index 12b48751bed5b..25b4a58c9026e 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionFieldDocValuesExtension.java @@ -26,7 +26,10 @@ public class VersionFieldDocValuesExtension implements PainlessExtension { - private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(VersionFieldDocValuesExtension.class, "whitelist.txt"); + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles( + VersionFieldDocValuesExtension.class, + "org.elasticsearch.xpack.versionfield.txt" + ); @Override public Map, List> getContextWhitelists() { diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java index f20db9bcd852e..3b145b65f17d3 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionScriptDocValues.java @@ -7,51 +7,11 @@ package org.elasticsearch.xpack.versionfield; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.index.fielddata.ScriptDocValues; -import java.io.IOException; - public final class VersionScriptDocValues extends ScriptDocValues { - public static final class VersionScriptSupplier implements ScriptDocValues.Supplier { - - private final SortedSetDocValues in; - private long[] ords = new long[0]; - private int count; - - public VersionScriptSupplier(SortedSetDocValues in) { - this.in = in; - } - - @Override - public void setNextDocId(int docId) throws IOException { - count = 0; - if (in.advanceExact(docId)) { - for (long ord = in.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = in.nextOrd()) { - ords = ArrayUtil.grow(ords, count + 1); - ords[count++] = ord; - } - } - } - - @Override - public String getInternal(int index) { - try { - return VersionEncoder.decodeVersion(in.lookupOrd(ords[index])); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public int size() { - return count; - } - } - - public VersionScriptDocValues(VersionScriptSupplier supplier) { + public VersionScriptDocValues(Supplier supplier) { super(supplier); } diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringDocValuesField.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringDocValuesField.java new file mode 100644 index 0000000000000..6aa38f0cf3857 --- /dev/null +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringDocValuesField.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.versionfield; + +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.field.DocValuesField; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +public class VersionStringDocValuesField implements DocValuesField, ScriptDocValues.Supplier { + + protected final SortedSetDocValues input; + protected final String name; + + protected long[] ords = new long[0]; + protected int count; + + // used for backwards compatibility for old-style "doc" access + // as a delegate to this field class + private VersionScriptDocValues versionScriptDocValues = null; + + public VersionStringDocValuesField(SortedSetDocValues input, String name) { + this.input = input; + this.name = name; + } + + @Override + public void setNextDocId(int docId) throws IOException { + count = 0; + if (input.advanceExact(docId)) { + for (long ord = input.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = input.nextOrd()) { + ords = ArrayUtil.grow(ords, count + 1); + ords[count++] = ord; + } + } + } + + @Override + public String getInternal(int index) { + try { + return VersionEncoder.decodeVersion(input.lookupOrd(ords[index])); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public int size() { + return count; + } + + @Override + public ScriptDocValues getScriptDocValues() { + if (versionScriptDocValues == null) { + versionScriptDocValues = new VersionScriptDocValues(this); + } + + return versionScriptDocValues; + } + + public String asString(String defaultValue) { + return asString(0, defaultValue); + } + + public String asString(int index, String defaultValue) { + if (isEmpty() || index < 0 || index >= size()) { + return defaultValue; + } + + return getInternal(index); + } + + public List asStrings() { + if (isEmpty()) { + return Collections.emptyList(); + } + + List values = new ArrayList<>(size()); + for (int i = 0; i < size(); i++) { + values.add(getInternal(i)); + } + + return values; + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isEmpty() { + return count == 0; + } + + public Version get(Version defaultValue) { + return get(0, defaultValue); + } + + public Version get(int index, Version defaultValue) { + if (isEmpty() || index < 0 || index >= size()) { + return defaultValue; + } + + return new Version(getInternal(index)); + } + + /** + * Returns an iterator over elements of type {@code T}. + * + * @return an Iterator. + */ + @Override + public Iterator iterator() { + return new Iterator() { + private int index = 0; + + @Override + public boolean hasNext() { + return index < size(); + } + + @Override + public Version next() { + if (hasNext() == false) { + throw new NoSuchElementException(); + } + return new Version(getInternal(index++)); + } + }; + } +} diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 5b5044bb0c930..a8c94034de585 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -44,7 +44,6 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.support.QueryParsers; -import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; @@ -63,7 +62,6 @@ import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; import static org.elasticsearch.xpack.versionfield.VersionEncoder.encodeVersion; -import static org.elasticsearch.xpack.versionfield.VersionScriptDocValues.VersionScriptSupplier; /** * A {@link FieldMapper} for indexing fields with version strings. @@ -281,11 +279,7 @@ protected BytesRef indexedValueForSearch(Object value) { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - return new SortedSetOrdinalsIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - (dv, n) -> new DelegateDocValuesField(new VersionScriptDocValues(new VersionScriptSupplier(dv)), n) - ); + return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, VersionStringDocValuesField::new); } @Override diff --git a/x-pack/plugin/mapper-version/src/main/resources/org/elasticsearch/xpack/versionfield/org.elasticsearch.xpack.versionfield.txt b/x-pack/plugin/mapper-version/src/main/resources/org/elasticsearch/xpack/versionfield/org.elasticsearch.xpack.versionfield.txt new file mode 100644 index 0000000000000..8e1a9dcc1f803 --- /dev/null +++ b/x-pack/plugin/mapper-version/src/main/resources/org/elasticsearch/xpack/versionfield/org.elasticsearch.xpack.versionfield.txt @@ -0,0 +1,24 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +class org.elasticsearch.xpack.versionfield.VersionScriptDocValues { + String get(int) + String getValue() +} + +class org.elasticsearch.xpack.versionfield.Version { + (String) +} + +class org.elasticsearch.xpack.versionfield.VersionStringDocValuesField @dynamic_type { + String asString(String) + String asString(int, String) + List asStrings() + Version get(Version) + Version get(int, Version) +} diff --git a/x-pack/plugin/mapper-version/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt b/x-pack/plugin/mapper-version/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt deleted file mode 100644 index eb47eae53a158..0000000000000 --- a/x-pack/plugin/mapper-version/src/main/resources/org/elasticsearch/xpack/versionfield/whitelist.txt +++ /dev/null @@ -1,5 +0,0 @@ - -class org.elasticsearch.xpack.versionfield.VersionScriptDocValues { - String get(int) - String getValue() -} \ No newline at end of file diff --git a/x-pack/plugin/mapper-version/src/yamlRestTest/java/org/elasticsearch/xpack/versionfield/VersionClientYamlTestSuiteIT.java b/x-pack/plugin/mapper-version/src/yamlRestTest/java/org/elasticsearch/xpack/versionfield/VersionClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..bc9f32766a3bb --- /dev/null +++ b/x-pack/plugin/mapper-version/src/yamlRestTest/java/org/elasticsearch/xpack/versionfield/VersionClientYamlTestSuiteIT.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.versionfield; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +/** Runs yaml rest tests */ +public class VersionClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public VersionClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/50_script_values.yml b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/50_script_values.yml new file mode 100644 index 0000000000000..806735e8174cf --- /dev/null +++ b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/50_script_values.yml @@ -0,0 +1,45 @@ +setup: + + - skip: + version: " - 8.0.99" + reason: "version script field support was added in 8.1.0" + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + ver: + type: version + + - do: + bulk: + index: test1 + refresh: true + body: | + { "index": {"_id" : "1"} } + { "ver": "1.0.0" } + { "index": {"_id" : "2"} } + { "ver": "1.2.3-abc+def" } + { "index": {"_id" : "3"} } + { "ver": "1.2.3.4.5" } + { "index": {"_id" : "4"} } + { "ver": ["6.7.8", "5.4.3"] } + +--- +"Scripted fields version values return Version": + - do: + search: + index: test1 + body: + sort: [ { ver: desc } ] + script_fields: + field: + script: + source: "field('ver').get(new Version(''))" + + - match: { hits.hits.0.fields.field.0: "5.4.3" } + - match: { hits.hits.1.fields.field.0: "1.2.3.4.5" } + - match: { hits.hits.2.fields.field.0: "1.2.3-abc+def" } + - match: { hits.hits.3.fields.field.0: "1.0.0" }