diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index aa4daf3179adb..bf8e8f57ba3ec 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -19,8 +19,8 @@ package org.elasticsearch.script.expression; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -64,7 +64,7 @@ public void setUp() throws Exception { when(fieldData.load(anyObject())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, ignored -> fieldData); + lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData); } private FieldScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index 007f4d608c82d..5410d95956c44 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -19,8 +19,8 @@ package org.elasticsearch.script.expression; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; @@ -64,7 +64,7 @@ public void setUp() throws Exception { when(fieldData.load(anyObject())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, ignored -> fieldData); + lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData); } private NumberSortScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index fd279a9fa14e9..80bddabc67f60 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -19,8 +19,8 @@ package org.elasticsearch.script.expression; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; @@ -64,7 +64,7 @@ public void setUp() throws Exception { when(fieldData.load(anyObject())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, ignored -> fieldData); + lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData); } private TermsSetQueryScript.LeafFactory compile(String expression) { diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index 92816426eaf47..e9a6ca60509e3 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.script.NumberSortScript; import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import java.util.Collections; @@ -47,23 +46,21 @@ public void testNeedsScores() { PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null); - SearchLookup lookup = new SearchLookup(index.mapperService(), shardContext::getForField); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); - NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), lookup); + NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); assertFalse(ss.needs_score()); factory = service.compile(null, "doc['d'].value", NumberSortScript.CONTEXT, Collections.emptyMap()); - ss = factory.newFactory(Collections.emptyMap(), lookup); + ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); assertFalse(ss.needs_score()); factory = service.compile(null, "1/_score", NumberSortScript.CONTEXT, Collections.emptyMap()); - ss = factory.newFactory(Collections.emptyMap(), lookup); + ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); assertTrue(ss.needs_score()); factory = service.compile(null, "doc['d'].value * _score", NumberSortScript.CONTEXT, Collections.emptyMap()); - ss = factory.newFactory(Collections.emptyMap(), lookup); + ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); assertTrue(ss.needs_score()); } - } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index a33df960db406..3513406d2ae75 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -31,11 +31,13 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * A {@link FieldMapper} that exposes Lucene's {@link FeatureField}. @@ -118,7 +120,7 @@ public Query existsQuery(QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { throw new IllegalArgumentException("[rank_feature] fields do not support sorting, scripting or aggregating"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index a088c6f70f793..59b8ccb7f05f9 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -27,10 +27,12 @@ import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * A {@link FieldMapper} that exposes Lucene's {@link FeatureField} as a sparse @@ -91,7 +93,7 @@ public Query existsQuery(QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { throw new IllegalArgumentException("[rank_features] fields do not support sorting, scripting or aggregating"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 597c39a3cc6ed..6276042c0f31c 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.math.BigDecimal; @@ -56,6 +57,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** A {@link FieldMapper} for scaled floats. Values are internally multiplied * by a scaling factor and rounded to the closest long. */ @@ -215,7 +217,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return (cache, breakerService, mapperService) -> { final IndexNumericFieldData scaledValues = new SortedNumericIndexFieldData.Builder( diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java index 64d7115196c0f..e893198a0cc5f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -150,7 +150,9 @@ public void testFieldData() throws IOException { // single-valued ScaledFloatFieldMapper.ScaledFloatFieldType f1 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float1", scalingFactor); - IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder("index").build(null, null, null); + IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder("index", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, null); assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE); LeafNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0)); SortedNumericDoubleValues values = leafFieldData.getDoubleValues(); @@ -161,7 +163,9 @@ public void testFieldData() throws IOException { // multi-valued ScaledFloatFieldMapper.ScaledFloatFieldType f2 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float2", scalingFactor); - fielddata = (IndexNumericFieldData) f2.fielddataBuilder("index").build(null, null, null); + fielddata = (IndexNumericFieldData) f2.fielddataBuilder("index", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, null); leafFieldData = fielddata.load(reader.leaves().get(0)); values = leafFieldData.getDoubleValues(); assertTrue(values.advanceExact(0)); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 590214dacdfd0..161c61137d8c5 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -33,10 +33,12 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.function.Supplier; /** * Simple field mapper hack to ensure that there is a one and only {@link ParentJoinFieldMapper} per mapping. @@ -90,7 +92,7 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 497139ca08138..b9bf5ade734c4 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -43,12 +43,14 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; /** * A field mapper used internally by the {@link ParentJoinFieldMapper} to index @@ -108,7 +110,7 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 058fefa9b5192..aef2495932ec0 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -34,13 +34,13 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; @@ -48,6 +48,7 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -58,6 +59,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Supplier; /** * A {@link FieldMapper} that creates hierarchical joins (parent-join) between documents in the same index. @@ -218,7 +220,7 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index c5f9dd3c7800d..b7959f3b5d8a6 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -617,7 +617,8 @@ public BitSetProducer bitsetFilter(Query query) { @Override @SuppressWarnings("unchecked") public > IFD getForField(MappedFieldType fieldType) { - IndexFieldData.Builder builder = fieldType.fielddataBuilder(shardContext.getFullyQualifiedIndex().getName()); + IndexFieldData.Builder builder = fieldType.fielddataBuilder(shardContext.getFullyQualifiedIndex().getName(), + shardContext::lookup); IndexFieldDataCache cache = new IndexFieldDataCache.None(); CircuitBreakerService circuitBreaker = new NoneCircuitBreakerService(); return (IFD) builder.build(cache, circuitBreaker, shardContext.getMapperService()); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index e3c05781a0e5e..50d19c3c6684d 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -46,6 +46,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -54,6 +55,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; public class ICUCollationKeywordFieldMapper extends FieldMapper { @@ -105,7 +107,7 @@ public Query existsQuery(QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 8156d5f9efa30..117b36366e22c 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -42,10 +42,12 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class Murmur3FieldMapper extends FieldMapper { @@ -105,7 +107,7 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 31834cd6a4e1b..ed26a3651ed89 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -199,7 +199,9 @@ public IndexService( // The sort order is validated right after the merge of the mapping later in the process. this.indexSortSupplier = () -> indexSettings.getIndexSortConfig().buildIndexSort( mapperService::fieldType, - indexFieldData::getForField + fieldType -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), () -> { + throw new UnsupportedOperationException("search lookup not available for index sorting"); + }) ); } else { this.indexSortSupplier = () -> null; diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index e9af9ce2bc67a..eb83e85bb7c0b 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -192,7 +192,7 @@ public Sort buildIndexSort(Function fieldTypeLookup, try { fieldData = fieldDataLookup.apply(ft); } catch (Exception e) { - throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]"); + throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]", e); } if (fieldData == null) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java index 3b3dc5353fd7a..49c1aeadf98d1 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java +++ b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java @@ -131,7 +131,10 @@ public TerminationHandle warmReader(final IndexShard indexShard, final Elasticse executor.execute(() -> { try { final long start = System.nanoTime(); - IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType); + IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType, indexFieldDataService.index().getName(), + () -> { + throw new UnsupportedOperationException("search lookup not available when warming an index"); + }); IndexFieldData global = ifd.loadGlobal(reader); if (reader.leaves().isEmpty() == false) { global.load(reader.leaves().get(0)); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 324728e8b090c..a0bf9b2082c3f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.Closeable; import java.io.IOException; @@ -38,6 +39,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class IndexFieldDataService extends AbstractIndexComponent implements Closeable { public static final String FIELDDATA_CACHE_VALUE_NODE = "node"; @@ -106,14 +108,16 @@ public synchronized void clearField(final String fieldName) { ExceptionsHelper.maybeThrowRuntimeAndSuppress(exceptions); } - public > IFD getForField(MappedFieldType fieldType) { - return getForField(fieldType, index().getName()); - } - + /** + * Returns fielddata for the provided field type, given the provided fully qualified index name, while also making + * a {@link SearchLookup} supplier available that is required for runtime fields. + */ @SuppressWarnings("unchecked") - public > IFD getForField(MappedFieldType fieldType, String fullyQualifiedIndexName) { + public > IFD getForField(MappedFieldType fieldType, + String fullyQualifiedIndexName, + Supplier searchLookup) { final String fieldName = fieldType.name(); - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName); + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName, searchLookup); IndexFieldDataCache cache; synchronized (this) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 3e50a4f21d8fd..863c352876b02 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -45,6 +46,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class BinaryFieldMapper extends ParametrizedFieldMapper { @@ -123,7 +125,7 @@ public BytesReference valueForDisplay(Object value) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new BytesBinaryIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 78e3e84562f09..6ddf61a970eb2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -39,12 +39,14 @@ import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * A field mapper for boolean fields. @@ -169,7 +171,7 @@ public Boolean valueForDisplay(Object value) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.BOOLEAN); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index a17d86a13efe0..b018f6a81b55e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.DateTimeException; @@ -61,6 +62,7 @@ import java.util.Map; import java.util.function.Function; import java.util.function.LongSupplier; +import java.util.function.Supplier; import static org.elasticsearch.common.time.DateUtils.toLong; @@ -435,7 +437,7 @@ public Function pointReaderIfPossible() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), resolution.numericType()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 0b5092ca370e1..fa6b510ccd0aa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -38,12 +38,14 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.VectorGeoPointShapeQueryProcessor; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Field Mapper for geo_point types. @@ -184,7 +186,7 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new AbstractLatLonPointIndexFieldData.Builder(name(), CoreValuesSourceType.GEOPOINT); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 1fbc2a459053d..3c08d3cdb4ead 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -46,6 +46,7 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; @@ -53,6 +54,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Supplier; /** * A mapper for the _id field. It does nothing since _id is neither indexed nor @@ -140,7 +142,7 @@ public Query termsQuery(List values, QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { final IndexFieldData.Builder fieldDataBuilder = new PagedBytesIndexFieldData.Builder( name(), TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 6b28bebe6d4f7..20cb3c8153eab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -25,10 +25,11 @@ import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; - +import java.util.function.Supplier; public class IndexFieldMapper extends MetadataFieldMapper { @@ -62,7 +63,7 @@ public Query existsQuery(QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { return new ConstantIndexFieldData.Builder(mapperService -> fullyQualifiedIndexName, name(), CoreValuesSourceType.BYTES); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index c922a26f6a92d..5f8e16cb2ef3f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.net.InetAddress; @@ -48,6 +49,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** A {@link FieldMapper} for ip addresses. */ public class IpFieldMapper extends ParametrizedFieldMapper { @@ -268,7 +270,7 @@ public int size() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), IpScriptDocValues::new, CoreValuesSourceType.IP); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 4b94d27a1a1b5..6fbe3322e702c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.UncheckedIOException; @@ -47,6 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. @@ -245,7 +247,7 @@ public Query existsQuery(QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index dd538cdb0125f..cd1e229c4aae4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -55,6 +56,7 @@ import java.util.Map; import java.util.Objects; import java.util.function.Function; +import java.util.function.Supplier; /** * This defines the core properties and functions to operate on a field. @@ -83,12 +85,12 @@ public MappedFieldType(String name, boolean isIndexed, boolean hasDocValues, Tex * Return a fielddata builder for this field * * @param fullyQualifiedIndexName the name of the index this field-data is build for - * + * @param searchLookup a {@link SearchLookup} supplier to allow for accessing other fields values in the context of runtime fields * @throws IllegalArgumentException if the fielddata is not supported on this type. * An IllegalArgumentException is needed in order to return an http error 400 * when this error occurs in a request. see: {@link org.elasticsearch.ExceptionsHelper#status} */ - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]"); } @@ -154,7 +156,9 @@ public Function pointReaderIfPossible() { */ public boolean isAggregatable() { try { - fielddataBuilder(""); + fielddataBuilder("", () -> { + throw new UnsupportedOperationException("SearchLookup not available"); + }); return true; } catch (IllegalArgumentException e) { return false; @@ -214,7 +218,7 @@ public Query wildcardQuery(String value, + "] which is of type [" + typeName() + "]"); } - public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxDeterminizedStates, + public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) { throw new QueryShardException(context, "Can only use regexp queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 0725b996c3523..1edea7ca6c04b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -53,6 +53,7 @@ import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -63,6 +64,7 @@ import java.util.Map; import java.util.Objects; import java.util.function.Function; +import java.util.function.Supplier; /** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */ public class NumberFieldMapper extends ParametrizedFieldMapper { @@ -927,7 +929,7 @@ public Function pointReaderIfPossible() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), type.numericType()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index a3bc8b25cb8f5..f0b3c095edf7b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -45,6 +45,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.net.InetAddress; @@ -60,6 +61,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Supplier; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD; @@ -214,7 +216,7 @@ public RangeFieldType(String name, DateFormatter formatter) { public RangeType rangeType() { return rangeType; } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new BinaryIndexFieldData.Builder(name(), CoreValuesSourceType.RANGE); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 9ec745bead343..a1137879c78c6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -33,11 +33,13 @@ import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.function.Supplier; /** * Mapper for the {@code _seq_no} field. @@ -167,7 +169,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 982a38180ce4e..690aca26f3bbd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -72,6 +72,7 @@ import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -82,6 +83,7 @@ import java.util.Map; import java.util.Objects; import java.util.function.IntPredicate; +import java.util.function.Supplier; import static org.elasticsearch.index.mapper.TypeParsers.checkNull; import static org.elasticsearch.index.mapper.TypeParsers.parseTextField; @@ -756,7 +758,7 @@ public static boolean hasGaps(TokenStream stream) throws IOException { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { if (fielddata == false) { throw new IllegalArgumentException("Text fields are not optimised for operations that require per-document " + "field data like aggregations and sorting, so these operations are disabled by default. Please use a " diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 9e6c3efb63cd2..6ee984fa5c2da 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Arrays; @@ -41,6 +42,7 @@ import java.util.HashSet; import java.util.Set; import java.util.function.Function; +import java.util.function.Supplier; public class TypeFieldMapper extends MetadataFieldMapper { @@ -69,7 +71,7 @@ public Query existsQuery(QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { Function typeFunction = mapperService -> mapperService.documentMapper().type(); return new ConstantIndexFieldData.Builder(typeFunction, name(), CoreValuesSourceType.BYTES); } diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index cc3a00bdd62a1..528831e3a11e8 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -31,6 +31,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.search.Queries; @@ -67,10 +68,10 @@ import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; -import java.util.function.BiFunction; import java.util.function.BooleanSupplier; import java.util.function.LongSupplier; import java.util.function.Predicate; +import java.util.function.Supplier; /** * Context object used to create lucene queries on the shard level. @@ -86,7 +87,7 @@ public class QueryShardContext extends QueryRewriteContext { private final MapperService mapperService; private final SimilarityService similarityService; private final BitsetFilterCache bitsetFilterCache; - private final BiFunction> indexFieldDataService; + private final TriFunction, IndexFieldData> indexFieldDataService; private final int shardId; private final IndexSearcher searcher; private boolean cacheable = true; @@ -106,7 +107,7 @@ public QueryShardContext(int shardId, IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, - BiFunction> indexFieldDataLookup, + TriFunction, IndexFieldData> indexFieldDataLookup, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService, @@ -136,7 +137,7 @@ private QueryShardContext(int shardId, IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, - BiFunction> indexFieldDataLookup, + TriFunction, IndexFieldData> indexFieldDataLookup, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService, @@ -208,7 +209,8 @@ public boolean allowExpensiveQueries() { @SuppressWarnings("unchecked") public > IFD getForField(MappedFieldType fieldType) { - return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName()); + return (IFD) indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), + () -> this.lookup().forkAndTrackFieldReferences(fieldType.name())); } public void addNamedQuery(String name, Query query) { @@ -289,11 +291,12 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap private SearchLookup lookup = null; public SearchLookup lookup() { - if (lookup == null) { - lookup = new SearchLookup(getMapperService(), - mappedFieldType -> indexFieldDataService.apply(mappedFieldType, fullyQualifiedIndex.getName())); + if (this.lookup == null) { + this.lookup = new SearchLookup( + getMapperService(), + (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup)); } - return lookup; + return this.lookup; } public NestedScope nestedScope() { diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 04aef7d2e8f63..fb0c81d29b4c7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -24,20 +24,86 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import java.util.function.Function; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Supplier; public class SearchLookup { + /** + * The maximum depth of field dependencies. + * When a runtime field's doc values depends on another runtime field's doc values, + * which depends on another runtime field's doc values and so on, it can + * make a very deep stack, which we want to limit. + */ + private static final int MAX_FIELD_CHAIN_DEPTH = 5; - final DocLookup docMap; + /** + * The chain of fields for which this lookup was created, used for detecting + * loops caused by runtime fields referring to other runtime fields. The chain is empty + * for the "top level" lookup created for the entire search. When a lookup is used to load + * fielddata for a field, we fork it and make sure the field name name isn't in the chain, + * then add it to the end. So the lookup for the a field named {@code a} will be {@code ["a"]}. If + * that field looks up the values of a field named {@code b} then + * {@code b}'s chain will contain {@code ["a", "b"]}. + */ + private final Set fieldChain; + private final DocLookup docMap; + private final SourceLookup sourceLookup; + private final FieldsLookup fieldsLookup; + private final BiFunction, IndexFieldData> fieldDataLookup; - final SourceLookup sourceLookup; - - final FieldsLookup fieldsLookup; - - public SearchLookup(MapperService mapperService, Function> fieldDataLookup) { - docMap = new DocLookup(mapperService, fieldDataLookup); + /** + * Create the top level field lookup for a search request. Provides a way to look up fields from doc_values, + * stored fields, or _source. + */ + public SearchLookup(MapperService mapperService, + BiFunction, IndexFieldData> fieldDataLookup) { + this.fieldChain = Collections.emptySet(); + docMap = new DocLookup(mapperService, + fieldType -> fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name()))); sourceLookup = new SourceLookup(); fieldsLookup = new FieldsLookup(mapperService); + this.fieldDataLookup = fieldDataLookup; + } + + /** + * Create a new {@link SearchLookup} that looks fields up the same as the one provided as argument, + * while also tracking field references starting from the provided field name. It detects cycles + * and prevents resolving fields that depend on more than {@link #MAX_FIELD_CHAIN_DEPTH} fields. + * @param searchLookup the existing lookup to create a new one from + * @param fieldChain the chain of fields that required the field currently being loaded + */ + private SearchLookup(SearchLookup searchLookup, Set fieldChain) { + this.fieldChain = Collections.unmodifiableSet(fieldChain); + this.docMap = new DocLookup(searchLookup.docMap.mapperService(), + fieldType -> searchLookup.fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name()))); + this.sourceLookup = searchLookup.sourceLookup; + this.fieldsLookup = searchLookup.fieldsLookup; + this.fieldDataLookup = searchLookup.fieldDataLookup; + } + + /** + * Creates a copy of the current {@link SearchLookup} that looks fields up in the same way, but also tracks field references + * in order to detect cycles and prevent resolving fields that depend on more than {@link #MAX_FIELD_CHAIN_DEPTH} other fields. + * @param field the field being referred to, for which fielddata needs to be loaded + * @return the new lookup + * @throws IllegalArgumentException if a cycle is detected in the fields required to build doc values, or if the field + * being resolved depends on more than {@link #MAX_FIELD_CHAIN_DEPTH} + */ + public final SearchLookup forkAndTrackFieldReferences(String field) { + Objects.requireNonNull(field, "field cannot be null"); + Set newFieldChain = new LinkedHashSet<>(fieldChain); + if (newFieldChain.add(field) == false) { + String message = String.join(" -> ", newFieldChain) + " -> " + field; + throw new IllegalArgumentException("Cyclic dependency detected while resolving runtime fields: " + message); + } + if (newFieldChain.size() > MAX_FIELD_CHAIN_DEPTH) { + throw new IllegalArgumentException("Field requires resolving too many dependent fields: " + String.join(" -> ", newFieldChain)); + } + return new SearchLookup(this, newFieldChain); } public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 5eed34cfe7287..ffc08f9e1d689 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -19,15 +19,11 @@ package org.elasticsearch.index; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; -import java.io.IOException; - import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.hamcrest.Matchers.containsString; @@ -35,28 +31,15 @@ public class IndexSortSettingsTests extends ESTestCase { private static IndexSettings indexSettings(Settings settings) { - return indexSettings(settings, null); - } - - private static IndexSettings indexSettings(Settings settings, Version version) { - final Settings newSettings; - if (version != null) { - newSettings = Settings.builder() - .put(settings) - .put(IndexMetadata.SETTING_VERSION_CREATED, version) - .build(); - } else { - newSettings = settings; - } - return new IndexSettings(newIndexMeta("test", newSettings), Settings.EMPTY); + return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); } - public void testNoIndexSort() throws IOException { + public void testNoIndexSort() { IndexSettings indexSettings = indexSettings(EMPTY_SETTINGS); assertFalse(indexSettings.getIndexSortConfig().hasIndexSort()); } - public void testSimpleIndexSort() throws IOException { + public void testSimpleIndexSort() { Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.order", "asc") @@ -74,7 +57,7 @@ public void testSimpleIndexSort() throws IOException { assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MAX)); } - public void testIndexSortWithArrays() throws IOException { + public void testIndexSortWithArrays() { Settings settings = Settings.builder() .putList("index.sort.field", "field1", "field2") .putList("index.sort.order", "asc", "desc") @@ -95,7 +78,7 @@ public void testIndexSortWithArrays() throws IOException { assertNull(config.sortSpecs[1].mode); } - public void testInvalidIndexSort() throws IOException { + public void testInvalidIndexSort() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.order", "asc, desc") @@ -105,7 +88,7 @@ public void testInvalidIndexSort() throws IOException { assertThat(exc.getMessage(), containsString("index.sort.field:[field1] index.sort.order:[asc, desc], size mismatch")); } - public void testInvalidIndexSortWithArray() throws IOException { + public void testInvalidIndexSortWithArray() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .putList("index.sort.order", new String[] {"asc", "desc"}) @@ -116,7 +99,7 @@ public void testInvalidIndexSortWithArray() throws IOException { containsString("index.sort.field:[field1] index.sort.order:[asc, desc], size mismatch")); } - public void testInvalidOrder() throws IOException { + public void testInvalidOrder() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.order", "invalid") @@ -126,7 +109,7 @@ public void testInvalidOrder() throws IOException { assertThat(exc.getMessage(), containsString("Illegal sort order:invalid")); } - public void testInvalidMode() throws IOException { + public void testInvalidMode() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.mode", "invalid") @@ -136,7 +119,7 @@ public void testInvalidMode() throws IOException { assertThat(exc.getMessage(), containsString("Illegal sort mode: invalid")); } - public void testInvalidMissing() throws IOException { + public void testInvalidMissing() { final Settings settings = Settings.builder() .put("index.sort.field", "field1") .put("index.sort.missing", "default") diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 25d3a1f1c772c..579c6e4ad6394 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -30,6 +30,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; @@ -46,18 +47,23 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; +import org.mockito.Matchers; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class IndexFieldDataServiceTests extends ESSingleNodeTestCase { @@ -74,7 +80,9 @@ public void testGetForFieldDefaults() { final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1)); final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string").build(ctx).fieldType(); ifdService.clear(); - IndexFieldData fd = ifdService.getForField(stringMapper); + IndexFieldData fd = ifdService.getForField(stringMapper, "test", () -> { + throw new UnsupportedOperationException(); + }); assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData); for (MappedFieldType mapper : Arrays.asList( @@ -84,23 +92,47 @@ public void testGetForFieldDefaults() { new NumberFieldMapper.Builder("long", NumberFieldMapper.NumberType.LONG, false, true).build(ctx).fieldType() )) { ifdService.clear(); - fd = ifdService.getForField(mapper); + fd = ifdService.getForField(mapper, "test", () -> { + throw new UnsupportedOperationException(); + }); assertTrue(fd instanceof SortedNumericIndexFieldData); } final MappedFieldType floatMapper = new NumberFieldMapper.Builder("float", NumberFieldMapper.NumberType.FLOAT, false, true) .build(ctx).fieldType(); ifdService.clear(); - fd = ifdService.getForField(floatMapper); + fd = ifdService.getForField(floatMapper, "test", () -> { + throw new UnsupportedOperationException(); + }); assertTrue(fd instanceof SortedNumericIndexFieldData); final MappedFieldType doubleMapper = new NumberFieldMapper.Builder("double", NumberFieldMapper.NumberType.DOUBLE, false, true) .build(ctx).fieldType(); ifdService.clear(); - fd = ifdService.getForField(doubleMapper); + fd = ifdService.getForField(doubleMapper, "test", () -> { + throw new UnsupportedOperationException(); + }); assertTrue(fd instanceof SortedNumericIndexFieldData); } + public void testGetForFieldRuntimeField() { + final IndexService indexService = createIndex("test"); + final IndicesService indicesService = getInstanceFromNode(IndicesService.class); + final IndexFieldDataService ifdService = new IndexFieldDataService(indexService.getIndexSettings(), + indicesService.getIndicesFieldDataCache(), indicesService.getCircuitBreakerService(), indexService.mapperService()); + final SetOnce> searchLookupSetOnce = new SetOnce<>(); + MappedFieldType ft = mock(MappedFieldType.class); + when(ft.fielddataBuilder(Matchers.any(), Matchers.any())).thenAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + Supplier searchLookup = (Supplier)invocationOnMock.getArguments()[1]; + searchLookupSetOnce.set(searchLookup); + return (IndexFieldData.Builder) (cache, breakerService, mapperService) -> null; + }); + SearchLookup searchLookup = new SearchLookup(null, null); + ifdService.getForField(ft, "qualified", () -> searchLookup); + assertSame(searchLookup, searchLookupSetOnce.get().get()); + } + public void testClearField() throws Exception { final IndexService indexService = createIndex("test"); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -130,8 +162,12 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon onRemovalCalled.incrementAndGet(); } }); - IndexFieldData ifd1 = ifdService.getForField(mapper1); - IndexFieldData ifd2 = ifdService.getForField(mapper2); + IndexFieldData ifd1 = ifdService.getForField(mapper1, "test", () -> { + throw new UnsupportedOperationException(); + }); + IndexFieldData ifd2 = ifdService.getForField(mapper2, "test", () -> { + throw new UnsupportedOperationException(); + }); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData loadField1 = ifd1.load(leafReaderContext); LeafFieldData loadField2 = ifd2.load(leafReaderContext); @@ -200,7 +236,9 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon onRemovalCalled.incrementAndGet(); } }); - IndexFieldData ifd = ifdService.getForField(mapper1); + IndexFieldData ifd = ifdService.getForField(mapper1, "test", () -> { + throw new UnsupportedOperationException(); + }); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData load = ifd.load(leafReaderContext); assertEquals(1, onCacheCalled.get()); @@ -256,10 +294,14 @@ private void doTestRequireDocValues(MappedFieldType ft) { IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), cache, null, null); if (ft.hasDocValues()) { - ifds.getForField(ft); // no exception + ifds.getForField(ft, "test", () -> { + throw new UnsupportedOperationException(); + }); // no exception } else { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ifds.getForField(ft)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ifds.getForField(ft, "test", () -> { + throw new UnsupportedOperationException(); + })); assertThat(e.getMessage(), containsString("doc values")); } } finally { @@ -283,5 +325,4 @@ public void testRequireDocValuesOnBools() { doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field")); doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field", true, false, Collections.emptyMap())); } - } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java index dac12c9fee555..4b425c378c1a6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java @@ -83,14 +83,18 @@ public void testEnableFieldData() throws IOException { IdFieldMapper.IdFieldType ft = (IdFieldMapper.IdFieldType) service.mapperService().fieldType("_id"); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> ft.fielddataBuilder("test").build(null, null, mapperService)); + () -> ft.fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, mapperService)); assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())); client().admin().cluster().prepareUpdateSettings() .setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true)) .get(); try { - ft.fielddataBuilder("test").build(null, null, mapperService); + ft.fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, mapperService); assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE); } finally { // unset cluster setting diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index b770bb93da6f4..274c764478ecc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -451,7 +451,9 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS // Create an index writer configured with the same index sort. NumberFieldType fieldType = new NumberFieldType("field", type); - IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder("index").build(null, null, null); + IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder("index", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, null); SortField sortField = fielddata.sortField(null, MultiValueMode.MIN, null, randomBoolean()); IndexWriterConfig writerConfig = new IndexWriterConfig(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 0fa6c03f76abd..1c54b2a7d1687 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -421,12 +421,16 @@ public void testFielddata() throws IOException { MapperService disabledMapper = createMapperService(fieldMapping(this::minimalMapping)); Exception e = expectThrows( IllegalArgumentException.class, - () -> disabledMapper.fieldType("field").fielddataBuilder("test") + () -> disabledMapper.fieldType("field").fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }) ); assertThat(e.getMessage(), containsString("Text fields are not optimised for operations that require per-document field data")); MapperService enabledMapper = createMapperService(fieldMapping(b -> b.field("type", "text").field("fielddata", true))); - enabledMapper.fieldType("field").fielddataBuilder("test"); // no exception this time + enabledMapper.fieldType("field").fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }); // no exception this time e = expectThrows( MapperParsingException.class, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java index 887ae9578df74..da37ad545674b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java @@ -30,9 +30,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.plugins.Plugin; @@ -68,8 +68,9 @@ public static void testDocValues(Function createIndex) thr w.close(); MappedFieldType ft = mapperService.fieldType(TypeFieldMapper.NAME); - IndexOrdinalsFieldData fd = (IndexOrdinalsFieldData) ft.fielddataBuilder("test") - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService(), mapperService); + IndexOrdinalsFieldData fd = (IndexOrdinalsFieldData) ft.fielddataBuilder("test", () -> { + throw new UnsupportedOperationException(); + }).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService(), mapperService); LeafOrdinalsFieldData afd = fd.load(r.leaves().get(0)); SortedSetDocValues values = afd.getOrdinalsValues(); assertTrue(values.advanceExact(0)); diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 93fe7e6e079d3..4a83530490041 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -18,9 +18,24 @@ */ package org.elasticsearch.index.query; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -29,21 +44,31 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.search.lookup.LeafDocLookup; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -153,7 +178,109 @@ null, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collec assertFalse(context.indexSortedOnField("non_sort_field")); } + public void testFielddataLookupSelfReference() { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + //simulate a runtime field that depends on itself e.g. field: doc['field'] + return leafLookup.doc().get(field).toString(); + }); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("field", queryShardContext)); + assertEquals("Cyclic dependency detected while resolving runtime fields: field -> field", iae.getMessage()); + } + + public void testFielddataLookupLooseLoop() { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + //simulate a runtime field cycle: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['1'] + if (field.equals("4")) { + return leafLookup.doc().get("1").toString(); + } + return leafLookup.doc().get(Integer.toString(Integer.parseInt(field) + 1)).toString(); + }); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); + assertEquals("Cyclic dependency detected while resolving runtime fields: 1 -> 2 -> 3 -> 4 -> 1", iae.getMessage()); + } + + public void testFielddataLookupTerminatesInLoop() { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + //simulate a runtime field cycle: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['4'] + if (field.equals("4")) { + return leafLookup.doc().get("4").toString(); + } + return leafLookup.doc().get(Integer.toString(Integer.parseInt(field) + 1)).toString(); + }); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); + assertEquals("Cyclic dependency detected while resolving runtime fields: 1 -> 2 -> 3 -> 4 -> 4", iae.getMessage()); + } + + public void testFielddataLookupSometimesLoop() throws IOException { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + if (docId == 0) { + return field + "_" + docId; + } else { + assert docId == 1; + if (field.equals("field4")) { + return leafLookup.doc().get("field1").toString(); + } + int i = Integer.parseInt(field.substring(field.length() - 1)); + return leafLookup.doc().get("field" + (i + 1)).toString(); + } + }); + List values = collect("field1", queryShardContext, new TermQuery(new Term("indexed_field", "first"))); + assertEquals(List.of("field1_0"), values); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("field1", queryShardContext)); + assertEquals("Cyclic dependency detected while resolving runtime fields: field1 -> field2 -> field3 -> field4 -> field1", + iae.getMessage()); + } + + public void testFielddataLookupBeyondMaxDepth() { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + int i = Integer.parseInt(field); + return leafLookup.doc().get(Integer.toString(i + 1)).toString(); + }); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); + assertEquals("Field requires resolving too many dependent fields: 1 -> 2 -> 3 -> 4 -> 5 -> 6", iae.getMessage()); + } + + public void testFielddataLookupReferencesBelowMaxDepth() throws IOException { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + int i = Integer.parseInt(field.substring(field.length() - 1)); + if (i == 5) { + return "test"; + } else { + ScriptDocValues scriptDocValues = leafLookup.doc().get("field" + (i + 1)); + return scriptDocValues.get(0).toString() + docId; + } + }); + assertEquals(List.of("test0000", "test1111"), collect("field1", queryShardContext)); + } + + public void testFielddataLookupOneFieldManyReferences() throws IOException { + int numFields = randomIntBetween(5, 20); + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + if (field.equals("field")) { + StringBuilder value = new StringBuilder(); + for (int i = 0; i < numFields; i++) { + value.append(leafLookup.doc().get("field" + i).get(0)); + } + return value.toString(); + } else { + return "test" + docId; + } + }); + StringBuilder expectedFirstDoc = new StringBuilder(); + StringBuilder expectedSecondDoc = new StringBuilder(); + for (int i = 0; i < numFields; i++) { + expectedFirstDoc.append("test0"); + expectedSecondDoc.append("test1"); + } + assertEquals(List.of(expectedFirstDoc.toString(), expectedSecondDoc.toString()), collect("field", queryShardContext)); + } + public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { + return createQueryShardContext(indexUuid, clusterAlias, null); + } + + private static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias, + TriFunction runtimeDocValues) { IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder("index"); indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) @@ -165,12 +292,113 @@ public static QueryShardContext createQueryShardContext(String indexUuid, String MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.index()).thenReturn(indexMetadata.getIndex()); + if (runtimeDocValues != null) { + when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { + String fieldName = (String)fieldTypeInv.getArguments()[0]; + return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); + }); + } final long nowInMillis = randomNonNegativeLong(); - return new QueryShardContext( 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, - (mappedFieldType, idxName) -> mappedFieldType.fielddataBuilder(idxName).build(null, null, null), + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null, null), mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); } + + private static MappedFieldType mockFieldType(String fieldName, BiFunction runtimeDocValues) { + MappedFieldType fieldType = mock(MappedFieldType.class); + when(fieldType.name()).thenReturn(fieldName); + when(fieldType.fielddataBuilder(any(), any())).thenAnswer(builderInv -> { + @SuppressWarnings("unchecked") + Supplier searchLookup = ((Supplier) builderInv.getArguments()[1]); + IndexFieldData indexFieldData = mock(IndexFieldData.class); + when(indexFieldData.load(any())).thenAnswer(loadArgs -> { + LeafReaderContext leafReaderContext = (LeafReaderContext) loadArgs.getArguments()[0]; + LeafFieldData leafFieldData = mock(LeafFieldData.class); + when(leafFieldData.getScriptValues()).thenAnswer(scriptValuesArgs -> new ScriptDocValues() { + String value; + + @Override + public int size() { + return 1; + } + + @Override + public String get(int index) { + assert index == 0; + return value; + } + + @Override + public void setNextDocId(int docId) { + assert docId >= 0; + LeafSearchLookup leafLookup = searchLookup.get().getLeafSearchLookup(leafReaderContext); + leafLookup.setDocument(docId); + value = runtimeDocValues.apply(leafLookup, docId); + } + }); + return leafFieldData; + }); + IndexFieldData.Builder builder = mock(IndexFieldData.Builder.class); + when(builder.build(any(), any(), any())).thenAnswer(buildInv -> indexFieldData); + return builder; + }); + return fieldType; + } + + private static List collect(String field, QueryShardContext queryShardContext) throws IOException { + return collect(field, queryShardContext, new MatchAllDocsQuery()); + } + + private static List collect(String field, QueryShardContext queryShardContext, Query query) throws IOException { + List result = new ArrayList<>(); + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexWriter.addDocument(List.of(new StringField("indexed_field", "first", Field.Store.NO))); + indexWriter.addDocument(List.of(new StringField("indexed_field", "second", Field.Store.NO))); + try (DirectoryReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newSearcher(reader); + MappedFieldType fieldType = queryShardContext.fieldMapper(field); + IndexFieldData indexFieldData; + if (randomBoolean()) { + indexFieldData = queryShardContext.getForField(fieldType); + } else { + indexFieldData = queryShardContext.lookup().doc().getForField(fieldType); + } + searcher.search(query, new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) { + ScriptDocValues scriptValues = indexFieldData.load(context).getScriptValues(); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) {} + + @Override + public void collect(int doc) throws IOException { + ScriptDocValues scriptDocValues; + if(randomBoolean()) { + LeafDocLookup leafDocLookup = queryShardContext.lookup().doc().getLeafDocLookup(context); + leafDocLookup.setDocument(doc); + scriptDocValues = leafDocLookup.get(field); + } else { + scriptDocValues = scriptValues; + } + scriptDocValues.setNextDocId(doc); + for (int i = 0; i < scriptDocValues.size(); i++) { + result.add(scriptDocValues.get(i).toString()); + } + } + }; + } + }); + } + return result; + } + } + } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index e7523d7b682a0..aaedf1f8ecc14 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2452,7 +2452,9 @@ public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException { new IndexFieldDataCache.Listener() {}); IndexFieldDataService indexFieldDataService = new IndexFieldDataService(shard.indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), shard.mapperService()); - IndexFieldData.Global ifd = indexFieldDataService.getForField(foo); + IndexFieldData.Global ifd = indexFieldDataService.getForField(foo, "test", () -> { + throw new UnsupportedOperationException("search lookup not available"); + }); FieldDataStats before = shard.fieldData().stats("foo"); assertThat(before.getMemorySizeInBytes(), equalTo(0L)); FieldDataStats after = null; diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index db91151e65e75..9a778459e6367 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.SortField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; @@ -56,6 +57,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.junit.AfterClass; @@ -65,8 +67,8 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; -import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.Supplier; import static java.util.Collections.emptyList; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; @@ -194,8 +196,9 @@ protected final QueryShardContext createMockShardContext(IndexSearcher searcher) IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build()); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class)); - BiFunction> indexFieldDataLookup = (fieldType, fieldIndexName) -> { - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName); + TriFunction, IndexFieldData> indexFieldDataLookup = + (fieldType, fieldIndexName, searchLookup) -> { + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null, null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 6b7f7cfcf5ca7..8eb5ad44b522a 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -49,6 +49,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; @@ -118,6 +119,7 @@ import org.elasticsearch.search.fetch.subphase.FetchSourcePhase; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalAggregationTestCase; import org.junit.After; @@ -132,9 +134,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.Collections.singletonList; @@ -313,8 +315,11 @@ public boolean shouldCache(Query query) { new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() { }), circuitBreakerService, mapperService); when(searchContext.getForField(Mockito.any(MappedFieldType.class))) - .thenAnswer(invocationOnMock -> ifds.getForField((MappedFieldType) invocationOnMock.getArguments()[0])); - + .thenAnswer(invocationOnMock -> ifds.getForField((MappedFieldType) invocationOnMock.getArguments()[0], + indexSettings.getIndex().getName(), + () -> { + throw new UnsupportedOperationException("search lookup not available"); + })); QueryShardContext queryShardContext = queryShardContextMock(contextIndexSearcher, mapperService, indexSettings, circuitBreakerService, bigArrays); when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); @@ -381,11 +386,11 @@ mapperService, null, getMockScriptService(), xContentRegistry(), /** * Sub-tests that need a more complex index field data provider can override this */ - protected BiFunction> getIndexFieldDataLookup(MapperService mapperService, - CircuitBreakerService circuitBreakerService) { - return (fieldType, s) -> fieldType.fielddataBuilder(mapperService.getIndexSettings().getIndex().getName()) + protected TriFunction, IndexFieldData> getIndexFieldDataLookup( + MapperService mapperService, CircuitBreakerService circuitBreakerService) { + return (fieldType, s, searchLookup) -> fieldType.fielddataBuilder( + mapperService.getIndexSettings().getIndex().getName(), searchLookup) .build(new IndexFieldDataCache.None(), circuitBreakerService, mapperService); - } /** @@ -707,7 +712,9 @@ public void testSupportedFieldTypes() throws IOException { } private ValuesSourceType fieldToVST(MappedFieldType fieldType) { - return fieldType.fielddataBuilder("").build(null, null, null).getValuesSourceType(); + return fieldType.fielddataBuilder("", () -> { + throw new UnsupportedOperationException(); + }).build(null, null, null).getValuesSourceType(); } /** diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index c9f8bf64df76e..3930a7c88189f 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -8,7 +8,6 @@ import com.carrotsearch.hppc.DoubleArrayList; import com.carrotsearch.hppc.IntArrayList; - import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; @@ -53,6 +52,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; @@ -61,6 +61,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -191,7 +192,7 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new IndexFieldData.Builder() { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 4678b6bf87295..d58c4685f3c52 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.time.ZoneId; @@ -47,6 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; /** * A {@link FieldMapper} that assigns every document the same value. @@ -140,7 +142,7 @@ public String familyTypeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { return new ConstantIndexFieldData.Builder(mapperService -> value, name(), CoreValuesSourceType.BYTES); } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 203bde6e37bad..c4283ac49a1b9 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -51,6 +51,7 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; @@ -59,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static org.elasticsearch.index.mapper.TypeParsers.parseField; @@ -319,7 +321,7 @@ public BytesRef indexedValueForSearch(Object value) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new KeyedFlatObjectFieldData.Builder(name(), key, CoreValuesSourceType.BYTES); } @@ -478,7 +480,7 @@ public Query existsQuery(QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java index 64b55b246f2de..420b0d99f2d81 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.lookup.SearchLookup; @@ -23,7 +23,8 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.function.Function; +import java.util.function.BiFunction; +import java.util.function.Supplier; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; @@ -161,7 +162,7 @@ public void testScriptDocValuesLookup() { = new KeyedFlatObjectFieldType( "field", true, true, "key2", false, Collections.emptyMap()); when(mapperService.fieldType("json.key2")).thenReturn(fieldType2); - Function> fieldDataSupplier = fieldType -> { + BiFunction, IndexFieldData> fieldDataSupplier = (fieldType, searchLookup) -> { KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) fieldType; return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; }; diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java index e7e1b0578fce2..917830355e4fb 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectIndexFieldDataTests.java @@ -80,7 +80,9 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { // Load global field data for subfield 'key'. KeyedFlatObjectFieldType fieldType1 = fieldMapper.keyedFieldType("key"); - IndexFieldData ifd1 = ifdService.getForField(fieldType1); + IndexFieldData ifd1 = ifdService.getForField(fieldType1, "test", () -> { + throw new UnsupportedOperationException("search lookup not available"); + }); assertTrue(ifd1 instanceof KeyedFlatObjectFieldData); KeyedFlatObjectFieldData fieldData1 = (KeyedFlatObjectFieldData) ifd1; @@ -90,7 +92,9 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { // Load global field data for the subfield 'other_key'. KeyedFlatObjectFieldType fieldType2 = fieldMapper.keyedFieldType("other_key"); - IndexFieldData ifd2 = ifdService.getForField(fieldType2); + IndexFieldData ifd2 = ifdService.getForField(fieldType2, "test", () -> { + throw new UnsupportedOperationException("search lookup not available"); + }); assertTrue(ifd2 instanceof KeyedFlatObjectFieldData); KeyedFlatObjectFieldData fieldData2 = (KeyedFlatObjectFieldData) ifd2; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 34750b8989027..adfa2914528bf 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.spatial.index.fielddata.AbstractLatLonShapeIndexFieldData; import org.elasticsearch.xpack.spatial.index.fielddata.CentroidCalculator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; @@ -37,6 +38,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Extension of {@link org.elasticsearch.index.mapper.GeoShapeFieldMapper} that supports docValues @@ -131,7 +133,7 @@ public GeoShapeWithDocValuesFieldType(String name, boolean indexed, boolean hasD super(name, indexed, hasDocValues, meta); } - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new AbstractLatLonShapeIndexFieldData.Builder(name(), GeoShapeValuesSourceType.instance()); } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 573bc5f0e5ac5..eeebb4e399d72 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.vectors.query.VectorIndexFieldData; import java.io.IOException; @@ -38,6 +39,7 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -128,7 +130,7 @@ public Query existsQuery(QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { return new VectorIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); } diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java index 403ec3f38e2cb..0d9b9f6d23922 100644 --- a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java @@ -16,6 +16,6 @@ public class SparseVectorFieldTypeTests extends FieldTypeTestCase { public void testDocValuesDisabled() { MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); - expectThrows(IllegalArgumentException.class, () -> fieldType.fielddataBuilder("index")); + expectThrows(IllegalArgumentException.class, () -> fieldType.fielddataBuilder("index", null)); } } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 250827d47ecd8..67152510e312d 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -66,6 +66,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -861,7 +862,7 @@ public Query termsQuery(List values, QueryShardContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); return new IndexFieldData.Builder() { @Override diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 507aef352758b..f137dac9262d5 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -41,6 +41,7 @@ //import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; @@ -56,6 +57,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -67,7 +69,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.function.BiFunction; +import java.util.function.Supplier; import static org.elasticsearch.index.mapper.FieldMapperTestCase.fetchSourceValue; import static org.hamcrest.Matchers.equalTo; @@ -812,8 +814,9 @@ protected final QueryShardContext createMockShardContext() { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build()); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class)); - BiFunction> indexFieldDataLookup = (fieldType, fieldIndexName) -> { - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName); + TriFunction, IndexFieldData> indexFieldDataLookup = + (fieldType, fieldIndexName, searchLookup) -> { + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null, null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup,