diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 4dd8f35c4812f..a54b70e231ab4 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -58,7 +58,6 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; /** @@ -85,7 +84,6 @@ public class ScriptScoreBenchmark { Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null)) ); private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None(); - private final Map> sourcePaths = Map.of("n", Set.of("n")); private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); private final SearchLookup lookup = new SearchLookup( fieldTypes::get, diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index 6b3410cabb899..ced2b900f5a16 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -24,7 +24,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.Objects; @@ -121,7 +120,7 @@ static class TokenCountFieldType extends NumberFieldMapper.NumberFieldType { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { if (hasDocValues() == false) { - return (lookup, doc, ignoredValues) -> List.of(); + return ValueFetcher.EMPTY; } return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index b3e6b5c6d5d2a..7e9b6916e99d4 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -29,7 +29,6 @@ import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import java.util.Collections; -import java.util.List; import java.util.Map; /** @@ -75,7 +74,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { // Although this is an internal field, we return it in the list of all field types. So we // provide an empty value fetcher here instead of throwing an error. - return (lookup, doc, ignoredValues) -> List.of(); + return ValueFetcher.EMPTY; } @Override diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index edc823c3d2f47..dd6e7035a1c17 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -21,7 +21,6 @@ import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; -import java.util.List; public class SizeFieldMapper extends MetadataFieldMapper { public static final String NAME = "_size"; @@ -57,7 +56,7 @@ private static class SizeFieldType extends NumberFieldType { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { if (hasDocValues() == false) { - return (lookup, doc, ignoredValues) -> List.of(); + return ValueFetcher.EMPTY; } return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java index febb0decd97bb..7321359b8cf33 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java @@ -46,7 +46,7 @@ public Status needsField(FieldInfo fieldInfo) { } private void addValue(Object value) { - destination.add(field.valueForDisplay(value)); + destination.add(value); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index ca3f5c0e1be5f..5ef372bd00c2c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -23,6 +23,7 @@ import org.elasticsearch.script.CompositeFieldScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xcontent.XContentBuilder; @@ -176,7 +177,11 @@ protected final void applyScriptContext(SearchExecutionContext context) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); + return new DocValueFetcher( + docValueFormat(format, null), + context.getForField(this, FielddataOperation.SEARCH), + StoredFieldsSpec.NEEDS_SOURCE // for now we assume runtime fields need source + ); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index 22dbb49726db8..4573463dc26cc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -10,6 +10,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; import java.util.ArrayList; @@ -71,6 +72,11 @@ public List fetchValues(Source source, int doc, List ignoredValu return values; } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } + /** * Given a value that has been extracted from a document's source, parse it into a standard * format. This parsing logic should closely mirror the value parsing in diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java index 6453f0ed9734c..eb4b1149836ab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java @@ -12,6 +12,7 @@ import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; import java.io.IOException; @@ -23,14 +24,22 @@ /** * Value fetcher that loads from doc values. */ +// TODO rename this? It doesn't load from doc values, it loads from fielddata +// Which might be doc values, but might not be... public final class DocValueFetcher implements ValueFetcher { private final DocValueFormat format; private final IndexFieldData ifd; private FormattedDocValues formattedDocValues; + private final StoredFieldsSpec storedFieldsSpec; - public DocValueFetcher(DocValueFormat format, IndexFieldData ifd) { + public DocValueFetcher(DocValueFormat format, IndexFieldData ifd, StoredFieldsSpec storedFieldsSpec) { this.format = format; this.ifd = ifd; + this.storedFieldsSpec = storedFieldsSpec; + } + + public DocValueFetcher(DocValueFormat format, IndexFieldData ifd) { + this(format, ifd, StoredFieldsSpec.NO_REQUIREMENTS); } @Override @@ -50,4 +59,8 @@ public List fetchValues(Source source, int doc, List ignoredValu return result; } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return storedFieldsSpec; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java index 6e21c777a5243..28789ed149200 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java @@ -28,6 +28,7 @@ import org.elasticsearch.script.GeoPointFieldScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.field.GeoPointDocValuesField; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.runtime.GeoPointScriptFieldDistanceFeatureQuery; @@ -215,6 +216,11 @@ public List fetchValues(Source source, int doc, List ignoredValu } return formatter.apply(points); } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } }; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index fdc2b9cef07d1..57bc7a96382fa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; import java.util.Collections; @@ -85,6 +86,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) public List fetchValues(Source source, int doc, List ignoredValues) { return indexName; } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } }; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LookupRuntimeFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/LookupRuntimeFieldType.java index beed384f4424a..7424c7be1f820 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LookupRuntimeFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LookupRuntimeFieldType.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.script.CompositeFieldScript; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.LookupField; import org.elasticsearch.search.lookup.SearchLookup; @@ -245,5 +246,10 @@ public DocumentField fetchDocumentField(String docName, Source source, int doc) public void setNextReader(LeafReaderContext context) { inputFieldValueFetcher.setNextReader(context); } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return inputFieldValueFetcher.storedFieldsSpec(); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java index 0d075e0bc11d2..a3c7c0d08e5cc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.fetch.subphase.FieldFetcher; import org.elasticsearch.search.lookup.Source; @@ -86,4 +87,9 @@ private Map createSourceMapStub(Map filteredSour public void setNextReader(LeafReaderContext context) { this.nestedFieldFetcher.setNextReader(context); } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index c82602f810561..20184db15581d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -10,6 +10,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; import java.util.ArrayDeque; @@ -93,6 +94,11 @@ public List fetchValues(Source source, int doc, List ignoredValu return values; } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } + /** * Given a value that has been extracted from a document's source, parse it into a standard * format. This parsing logic should closely mirror the value parsing in diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java index 10919f1cd4708..426ac66aba372 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/StoredValueFetcher.java @@ -9,12 +9,14 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import java.io.IOException; import java.util.List; +import java.util.Set; /** * Value fetcher that loads from stored values. @@ -24,10 +26,12 @@ public class StoredValueFetcher implements ValueFetcher { private final SearchLookup lookup; private LeafSearchLookup leafSearchLookup; private final String fieldname; + private final StoredFieldsSpec storedFieldsSpec; public StoredValueFetcher(SearchLookup lookup, String fieldname) { this.lookup = lookup; this.fieldname = fieldname; + this.storedFieldsSpec = new StoredFieldsSpec(false, false, Set.of(fieldname)); } @Override @@ -53,4 +57,8 @@ protected List parseStoredValues(List values) { return values; } + @Override + public StoredFieldsSpec storedFieldsSpec() { + return storedFieldsSpec; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java index 9ea41e4429a42..dfc1fac06e96b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.core.Nullable; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; import org.elasticsearch.search.lookup.Source; @@ -66,4 +67,35 @@ default DocumentField fetchDocumentField(String docName, Source lookup, int doc) * Update the leaf reader used to fetch values. */ default void setNextReader(LeafReaderContext context) {} + + /** + * The stored field or source requirements of this value fetcher + */ + StoredFieldsSpec storedFieldsSpec(); + + ValueFetcher EMPTY = new ValueFetcher() { + @Override + public List fetchValues(Source source, int doc, List ignoredValues) { + return List.of(); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + }; + + static ValueFetcher singleton(Object value) { + return new ValueFetcher() { + @Override + public List fetchValues(Source source, int doc, List ignoredValues) { + return List.of(value); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + }; + } } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index eb3c01c90c2ff..ea53234b55953 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -58,6 +58,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.NestedDocuments; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.lookup.LeafFieldLookupProvider; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.transport.RemoteClusterAware; @@ -494,19 +495,22 @@ public SearchLookup lookup() { SourceProvider sourceProvider = isSourceSynthetic() ? (ctx, doc) -> { throw new IllegalArgumentException("Cannot access source from scripts in synthetic mode"); } : SourceProvider.fromStoredFields(); - setSourceProvider(sourceProvider); + setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } return this.lookup; } /** - * Replace the standard source provider on the SearchLookup + * Replace the standard source provider and field lookup provider on the SearchLookup * * Note that this will replace the current SearchLookup with a new one, but will not update * the source provider on previously build lookups. This method should only be called before * IndexReader access by the current context */ - public void setSourceProvider(SourceProvider sourceProvider) { + public void setLookupProviders( + SourceProvider sourceProvider, + Function fieldLookupProvider + ) { // TODO can we assert that this is only called during FetchPhase? this.lookup = new SearchLookup( this::getFieldType, @@ -514,7 +518,8 @@ public void setSourceProvider(SourceProvider sourceProvider) { fieldType, new FieldDataContext(fullyQualifiedIndex.getName(), searchLookup, this::sourcePath, fielddataOperation) ), - sourceProvider + sourceProvider, + fieldLookupProvider ); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 62728adc753fc..e9c1ef4cea1bd 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1311,6 +1311,8 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc for (org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) { FieldScript.Factory factory = scriptService.compile(field.script(), FieldScript.CONTEXT); SearchLookup lookup = context.getSearchExecutionContext().lookup(); + // TODO delay this construction until the FetchPhase is executed so that we can + // use the more efficient lookup built there FieldScript.LeafFactory searchScript = factory.newFactory(field.script().getParams(), lookup); context.scriptFields().add(new ScriptField(field.fieldName(), searchScript, field.ignoreFailure())); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 0036c154231f2..e9884f81914fc 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -101,10 +101,7 @@ private SearchHits buildSearchHits(SearchContext context, Profiler profiler) { List processors = getProcessors(context.shardTarget(), fetchContext, profiler); - StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; - for (FetchSubPhaseProcessor proc : processors) { - storedFieldsSpec = storedFieldsSpec.merge(proc.storedFieldsSpec()); - } + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.build(processors, FetchSubPhaseProcessor::storedFieldsSpec); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(false, false, sourceLoader.requiredStoredFields())); StoredFieldLoader storedFieldLoader = profiler.storedFields(buildStoredFieldsLoader(storedFieldsSpec)); @@ -113,7 +110,8 @@ private SearchHits buildSearchHits(SearchContext context, Profiler profiler) { NestedDocuments nestedDocuments = context.getSearchExecutionContext().getNestedDocuments(); PreloadedSourceProvider sourceProvider = new PreloadedSourceProvider(); - context.getSearchExecutionContext().setSourceProvider(sourceProvider); + PreloadedFieldLookupProvider fieldLookupProvider = new PreloadedFieldLookupProvider(); + context.getSearchExecutionContext().setLookupProviders(sourceProvider, ctx -> fieldLookupProvider); FetchPhaseDocsIterator docsIterator = new FetchPhaseDocsIterator() { @@ -129,6 +127,7 @@ protected void setNextReader(LeafReaderContext ctx, int[] docsInLeaf) throws IOE this.leafNestedDocuments = nestedDocuments.getLeafNestedDocuments(ctx); this.leafStoredFieldLoader = storedFieldLoader.getLoader(ctx, docsInLeaf); this.leafSourceLoader = sourceLoader.leaf(ctx.reader(), docsInLeaf); + fieldLookupProvider.setNextReader(ctx); for (FetchSubPhaseProcessor processor : processors) { processor.setNextReader(ctx); } @@ -151,6 +150,7 @@ protected SearchHit nextDoc(int doc) throws IOException { leafSourceLoader ); sourceProvider.source = hit.source(); + fieldLookupProvider.storedFields = hit.loadedFields(); for (FetchSubPhaseProcessor processor : processors) { processor.process(hit); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java b/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java new file mode 100644 index 0000000000000..31cd74c878a0f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.fetch; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.search.lookup.FieldLookup; +import org.elasticsearch.search.lookup.LeafFieldLookupProvider; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Makes pre-loaded stored fields available via a LeafSearchLookup. + * + * If a stored field is requested that is not in the pre-loaded list, + * this loader will fall back to loading directly from the context + * stored fields + */ +class PreloadedFieldLookupProvider implements LeafFieldLookupProvider { + + Map> storedFields; + LeafFieldLookupProvider backUpLoader; + Supplier loaderSupplier; + + @Override + public void populateFieldLookup(FieldLookup fieldLookup, int doc) throws IOException { + String field = fieldLookup.fieldType().name(); + if (storedFields.containsKey(field)) { + fieldLookup.setValues(storedFields.get(field)); + return; + } + // stored field not preloaded, go and get it directly + if (backUpLoader == null) { + backUpLoader = loaderSupplier.get(); + } + backUpLoader.populateFieldLookup(fieldLookup, doc); + } + + void setNextReader(LeafReaderContext ctx) { + backUpLoader = null; + loaderSupplier = () -> LeafFieldLookupProvider.fromStoredFields().apply(ctx); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java index 0b68de0fc0242..45054a90c749f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -8,8 +8,10 @@ package org.elasticsearch.search.fetch; +import java.util.Collection; import java.util.HashSet; import java.util.Set; +import java.util.function.Function; /** * Defines which stored fields need to be loaded during a fetch @@ -44,4 +46,12 @@ public StoredFieldsSpec merge(StoredFieldsSpec other) { mergedFields ); } + + public static StoredFieldsSpec build(Collection sources, Function converter) { + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + for (T source : sources) { + storedFieldsSpec = storedFieldsSpec.merge(converter.apply(source)); + } + return storedFieldsSpec; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index c0bfad44195cd..b46f2752642a5 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -42,8 +42,7 @@ public void setNextReader(LeafReaderContext readerContext) { @Override public StoredFieldsSpec storedFieldsSpec() { - // TODO can we get finer-grained information from the FieldFetcher for this? - return StoredFieldsSpec.NEEDS_SOURCE; + return fieldFetcher.storedFieldsSpec(); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java index e28a261819100..cfbfe08a19ff8 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.NestedValueFetcher; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; import java.io.IOException; @@ -152,6 +153,7 @@ private static FieldFetcher create( private final Map fieldContexts; private final CharacterRunAutomaton unmappedFieldsFetchAutomaton; private final List unmappedConcreteFields; + private final StoredFieldsSpec storedFieldsSpec; private FieldFetcher( Map fieldContexts, @@ -161,6 +163,11 @@ private FieldFetcher( this.fieldContexts = fieldContexts; this.unmappedFieldsFetchAutomaton = unmappedFieldsFetchAutomaton; this.unmappedConcreteFields = unmappedConcreteFields; + this.storedFieldsSpec = StoredFieldsSpec.build(fieldContexts.values(), fc -> fc.valueFetcher.storedFieldsSpec()); + } + + public StoredFieldsSpec storedFieldsSpec() { + return storedFieldsSpec; } public Map fetch(Source source, int doc) throws IOException { @@ -291,13 +298,5 @@ public void setNextReader(LeafReaderContext readerContext) { } } - private static class FieldContext { - final String fieldName; - final ValueFetcher valueFetcher; - - FieldContext(String fieldName, ValueFetcher valueFetcher) { - this.fieldName = fieldName; - this.valueFetcher = valueFetcher; - } - } + private record FieldContext(String fieldName, ValueFetcher valueFetcher) {} } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java index 3aac60ca883c9..fa4eb8f21f78c 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java @@ -18,11 +18,11 @@ public class FieldLookup { private final List values = new ArrayList<>(); private boolean valuesLoaded = false; - FieldLookup(MappedFieldType fieldType) { + public FieldLookup(MappedFieldType fieldType) { this.fieldType = fieldType; } - MappedFieldType fieldType() { + public MappedFieldType fieldType() { return fieldType; } @@ -31,7 +31,7 @@ MappedFieldType fieldType() { */ public void setValues(List values) { assert valuesLoaded == false : "Call clear() before calling setValues()"; - this.values.addAll(values); + values.stream().map(fieldType::valueForDisplay).forEach(this.values::add); this.valuesLoaded = true; } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java b/server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java index 040d3b43a400a..19b362f1b6454 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.util.ArrayList; import java.util.List; @@ -67,6 +68,11 @@ public List fetchValues(Source lookup, int doc, List ignoredValu } return values; } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; // TODO can we get more information from the script + } }; } @@ -100,6 +106,11 @@ public List fetchValues(Source source, int doc, List ignoredValu } return formatter.apply(values); } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; // TODO can we get more info from the script? + } }; } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldLookupProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldLookupProvider.java new file mode 100644 index 0000000000000..a17bd1f2d26e5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafFieldLookupProvider.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.lookup; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.StoredFields; +import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +/** + * Defines how to populate the values of a {@link FieldLookup} + */ +public interface LeafFieldLookupProvider { + + /** + * Load stored field values for the given doc and cache them in the FieldLookup + */ + void populateFieldLookup(FieldLookup fieldLookup, int doc) throws IOException; + + /** + * Create a LeafFieldLookupProvider that loads values from stored fields + */ + static Function fromStoredFields() { + return ctx -> new LeafFieldLookupProvider() { + + StoredFields storedFields; + int currentDoc = -1; + final List currentValues = new ArrayList<>(2); + + @Override + public void populateFieldLookup(FieldLookup fieldLookup, int doc) throws IOException { + if (storedFields == null) { + storedFields = ctx.reader().storedFields(); + } + if (doc == currentDoc) { + fieldLookup.setValues(currentValues); + } else { + currentDoc = doc; + currentValues.clear(); + // TODO can we remember which fields have been loaded here and get them eagerly next time? + // likelihood is if a script is loading several fields on one doc they will load the same + // set of fields next time round + SingleFieldsVisitor visitor = new SingleFieldsVisitor(fieldLookup.fieldType(), currentValues); + storedFields.document(doc, visitor); + fieldLookup.setValues(currentValues); + } + } + }; + } + +} diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookup.java index eea6de3926d7c..8a60a27724d06 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookup.java @@ -7,17 +7,12 @@ */ package org.elasticsearch.search.lookup; -import org.apache.lucene.index.StoredFields; -import org.elasticsearch.common.CheckedSupplier; -import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor; import org.elasticsearch.index.mapper.MappedFieldType; import java.io.IOException; import java.io.UncheckedIOException; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -26,19 +21,15 @@ public class LeafStoredFieldsLookup implements Map { private final Function fieldTypeLookup; - private final CheckedSupplier storedFieldsSupplier; + private final LeafFieldLookupProvider leafFieldLookupProvider; private int docId = -1; - private StoredFields storedFields = null; private final Map cachedFieldData = new HashMap<>(); - LeafStoredFieldsLookup( - Function fieldTypeLookup, - CheckedSupplier storedFieldsSupplier - ) { + LeafStoredFieldsLookup(Function fieldTypeLookup, LeafFieldLookupProvider leafFieldLookupProvider) { this.fieldTypeLookup = fieldTypeLookup; - this.storedFieldsSupplier = storedFieldsSupplier; + this.leafFieldLookupProvider = leafFieldLookupProvider; } public void setDocument(int docId) { @@ -119,9 +110,6 @@ public boolean containsValue(Object value) { } private FieldLookup loadFieldData(String name) throws IOException { - if (storedFields == null) { - storedFields = storedFieldsSupplier.get(); - } FieldLookup data = cachedFieldData.get(name); if (data == null) { MappedFieldType fieldType = fieldTypeLookup.apply(name); @@ -132,10 +120,7 @@ private FieldLookup loadFieldData(String name) throws IOException { cachedFieldData.put(name, data); } if (data.isLoaded() == false) { - List values = new ArrayList<>(2); - SingleFieldsVisitor visitor = new SingleFieldsVisitor(data.fieldType(), values); - storedFields.document(docId, visitor); - data.setValues(values); + leafFieldLookupProvider.populateFieldLookup(data, docId); } return data; } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 9543c6479a676..06f71fbf2514d 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -21,6 +21,9 @@ import java.util.function.Function; import java.util.function.Supplier; +/** + * Provides a way to look up per-document values from docvalues, stored fields or _source + */ public class SearchLookup implements SourceProvider { /** * The maximum depth of field dependencies. @@ -47,20 +50,40 @@ public class SearchLookup implements SourceProvider { Supplier, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup; + private final Function fieldLookupProvider; /** - * Create the top level field lookup for a search request. Provides a way to look up fields from doc_values, - * stored fields, or _source. + * Create a new SearchLookup, using the default stored fields provider + * @param fieldTypeLookup defines how to look up field types + * @param fieldDataLookup defines how to look up field data + * @param sourceProvider defines how to look up the source */ public SearchLookup( Function fieldTypeLookup, TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup, SourceProvider sourceProvider + ) { + this(fieldTypeLookup, fieldDataLookup, sourceProvider, LeafFieldLookupProvider.fromStoredFields()); + } + + /** + * Create a new SearchLookup, using the default stored fields provider + * @param fieldTypeLookup defines how to look up field types + * @param fieldDataLookup defines how to look up field data + * @param sourceProvider defines how to look up the source + * @param fieldLookupProvider defines how to look up stored fields + */ + public SearchLookup( + Function fieldTypeLookup, + TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup, + SourceProvider sourceProvider, + Function fieldLookupProvider ) { this.fieldTypeLookup = fieldTypeLookup; this.fieldChain = Collections.emptySet(); this.sourceProvider = sourceProvider; this.fieldDataLookup = fieldDataLookup; + this.fieldLookupProvider = fieldLookupProvider; } /** @@ -75,6 +98,7 @@ private SearchLookup(SearchLookup searchLookup, Set fieldChain) { this.sourceProvider = searchLookup.sourceProvider; this.fieldTypeLookup = searchLookup.fieldTypeLookup; this.fieldDataLookup = searchLookup.fieldDataLookup; + this.fieldLookupProvider = searchLookup.fieldLookupProvider; } /** @@ -103,7 +127,7 @@ public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { context, new LeafDocLookup(fieldTypeLookup, this::getForField, context), sourceProvider, - new LeafStoredFieldsLookup(fieldTypeLookup, () -> context.reader().storedFields()) + new LeafStoredFieldsLookup(fieldTypeLookup, fieldLookupProvider.apply(context)) ); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldDataTests.java index d61cb5fba1a6d..15b6fca5c72f4 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldDataTests.java @@ -9,6 +9,9 @@ package org.elasticsearch.index.fielddata; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBooleanIndexFieldData.SourceValueFetcherSortedBooleanDocValues; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.Source; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -24,11 +27,17 @@ public void testSourceValueFetcherSortedBooleanDocValues() throws IOException { List.of(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()) ); - SourceValueFetcherSortedBooleanDocValues values = new SourceValueFetcherSortedBooleanDocValues( - null, - (source, doc, ignoredValues) -> docs.get(doc), - (ctx, doc) -> null - ); + SourceValueFetcherSortedBooleanDocValues values = new SourceValueFetcherSortedBooleanDocValues(null, new ValueFetcher() { + @Override + public List fetchValues(Source source, int doc, List ignoredValues) throws IOException { + return docs.get(doc); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + }, (ctx, doc) -> null); for (int doc = 0; doc < docs.size(); ++doc) { values.advanceExact(doc); diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 720c9d1363647..5eafdc0b6f0f7 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -70,6 +70,7 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.lookup.LeafDocLookup; +import org.elasticsearch.search.lookup.LeafFieldLookupProvider; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; @@ -412,7 +413,7 @@ public void testSyntheticSourceScriptLoading() throws IOException { // Setting the source provider explicitly then gives us a new SearchLookup that can use source Source source = Source.fromMap(Map.of("field", "value"), XContentType.JSON); - sec.setSourceProvider((ctx, doc) -> source); + sec.setLookupProviders((ctx, doc) -> source, LeafFieldLookupProvider.fromStoredFields()); SearchLookup searchLookup1 = sec.lookup(); assertNotSame(searchLookup, searchLookup1); assertSame(source, searchLookup1.getSource(null, 0)); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProviderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProviderTests.java new file mode 100644 index 0000000000000..85d9c32a1ee5b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProviderTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.fetch; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.search.lookup.FieldLookup; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PreloadedFieldLookupProviderTests extends ESTestCase { + + public void testFallback() throws IOException { + PreloadedFieldLookupProvider lookup = new PreloadedFieldLookupProvider(); + lookup.storedFields = Map.of("foo", List.of("bar")); + + MappedFieldType fieldType = mock(MappedFieldType.class); + when(fieldType.name()).thenReturn("foo"); + when(fieldType.valueForDisplay(any())).then(invocation -> ((String) invocation.getArguments()[0]).toUpperCase(Locale.ROOT)); + FieldLookup fieldLookup = new FieldLookup(fieldType); + + lookup.populateFieldLookup(fieldLookup, 0); + assertEquals("BAR", fieldLookup.getValue()); + assertNull(lookup.backUpLoader); // fallback didn't get used because 'foo' is in the list + + MappedFieldType unloadedFieldType = mock(MappedFieldType.class); + when(unloadedFieldType.name()).thenReturn("unloaded"); + when(unloadedFieldType.valueForDisplay(any())).then( + invocation -> ((BytesRef) invocation.getArguments()[0]).utf8ToString().toUpperCase(Locale.ROOT) + ); + FieldLookup unloadedFieldLookup = new FieldLookup(unloadedFieldType); + + MemoryIndex mi = new MemoryIndex(); + mi.addField(new StringField("unloaded", "value", Field.Store.YES), null); + LeafReaderContext ctx = mi.createSearcher().getIndexReader().leaves().get(0); + + lookup.setNextReader(ctx); + lookup.populateFieldLookup(unloadedFieldLookup, 0); + assertEquals("VALUE", unloadedFieldLookup.getValue()); + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 1a97d6dc37c7f..678b33f60be16 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.lookup.SourceProvider; @@ -1182,6 +1183,12 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException { }); } + public void testStoredFieldsSpec() throws IOException { + List fields = List.of(new FieldAndFormat("field", null)); + FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(createMapperService()), fields); + assertEquals(StoredFieldsSpec.NEEDS_SOURCE, fieldFetcher.storedFieldsSpec()); + } + private List fieldAndFormatList(String name, String format, boolean includeUnmapped) { return Collections.singletonList(new FieldAndFormat(name, format, includeUnmapped)); } diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java index dab82c4d92e7c..0ffe59f3cfd95 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafStoredFieldsLookupTests.java @@ -7,19 +7,9 @@ */ package org.elasticsearch.search.lookup; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.StoredFields; -import org.apache.lucene.index.VectorEncoding; -import org.apache.lucene.index.VectorSimilarityFunction; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.test.ESTestCase; -import org.junit.Before; -import java.io.IOException; -import java.util.Collections; import java.util.List; import static org.mockito.ArgumentMatchers.any; @@ -27,48 +17,21 @@ import static org.mockito.Mockito.when; public class LeafStoredFieldsLookupTests extends ESTestCase { - private LeafStoredFieldsLookup fieldsLookup; - - @Before - public void setUp() throws Exception { - super.setUp(); + private LeafStoredFieldsLookup buildFieldsLookup() { MappedFieldType fieldType = mock(MappedFieldType.class); when(fieldType.name()).thenReturn("field"); // Add 10 when valueForDisplay is called so it is easy to be sure it *was* called when(fieldType.valueForDisplay(any())).then(invocation -> (Double) invocation.getArguments()[0] + 10); - FieldInfo mockFieldInfo = new FieldInfo( - "field", - 1, - false, - false, - true, - IndexOptions.NONE, - DocValuesType.NONE, - -1, - Collections.emptyMap(), - 0, - 0, - 0, - 0, - VectorEncoding.FLOAT32, - VectorSimilarityFunction.EUCLIDEAN, - false - ); - - fieldsLookup = new LeafStoredFieldsLookup( + return new LeafStoredFieldsLookup( field -> field.equals("field") || field.equals("alias") ? fieldType : null, - () -> new StoredFields() { - @Override - public void document(int docID, StoredFieldVisitor visitor) throws IOException { - visitor.doubleField(mockFieldInfo, 2.718); - } - } + (fieldLookup, doc) -> fieldLookup.setValues(List.of(2.718)) ); } public void testBasicLookup() { + LeafStoredFieldsLookup fieldsLookup = buildFieldsLookup(); FieldLookup fieldLookup = fieldsLookup.get("field"); assertEquals("field", fieldLookup.fieldType().name()); @@ -79,6 +42,7 @@ public void testBasicLookup() { } public void testLookupWithFieldAlias() { + LeafStoredFieldsLookup fieldsLookup = buildFieldsLookup(); FieldLookup fieldLookup = fieldsLookup.get("alias"); assertEquals("field", fieldLookup.fieldType().name()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java index 9cfa9be606239..0d492fd837d7c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java @@ -22,7 +22,6 @@ import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; -import java.util.List; public class DataTierFieldMapper extends MetadataFieldMapper { @@ -79,9 +78,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } String tierPreference = getTierPreference(context); - return tierPreference == null - ? (lookup, doc, ignoredValues) -> List.of() - : (lookup, doc, ignoredValues) -> List.of(tierPreference); + return tierPreference == null ? ValueFetcher.EMPTY : ValueFetcher.singleton(tierPreference); } /** diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 5dd04a9e79645..45630fca55d9f 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -50,7 +50,6 @@ import java.io.IOException; import java.time.ZoneId; import java.util.Collections; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -150,7 +149,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value == null ? (lookup, doc, ignoredValues) -> List.of() : (lookup, doc, ignoredValues) -> List.of(value); + return value == null ? ValueFetcher.EMPTY : ValueFetcher.singleton(value); } @Override