From b34df5401b29b0a3fa64c05f20efcf096b88bc5b Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 9 Dec 2020 11:07:28 +0100 Subject: [PATCH 01/10] Sort field tiebreaker for PIT (point in time) readers This commit introduces a new sort field called `_shard_doc` that can be used in conjunction with a PIT to consistently tiebreak identical sort values. The sort value is a numeric long that is composed of the ordinal of the shard (assigned by the coordinating node) and the internal Lucene document ID. These two values are consistent within a PIT so this sort criteria can be used as the tiebreaker of any search requests. Since this sort criteria is stable we'd like to add it automatically to any sorted search requests that use a PIT but we also need to expose it explicitly in order to be able to: * Reverse the order of the tiebreaking, useful to search "before" `search_after`. * Force the primary sort to use it in order to benefit from the `search_after` optimization when sorting by index order (to be released in Lucene 8.8. I plan to add the documentation and the automatic configuration for PIT in a follow up since this change is already big. Relates #56828 --- .../common/DisableGraphQueryTests.java | 2 +- .../action/PainlessExecuteAction.java | 2 +- .../painless/NeedsScoreTests.java | 2 +- .../PercolatorFieldMapperTests.java | 2 +- .../PercolatorQuerySearchTests.java | 2 +- .../test/search/90_search_after.yml | 28 +++++- .../TransportSimulateIndexTemplateAction.java | 2 +- .../search/QueryPhaseResultConsumer.java | 10 ++- .../action/search/SearchRequest.java | 11 +++ .../metadata/MetadataCreateIndexService.java | 6 +- .../metadata/MetadataIndexAliasesService.java | 4 +- .../MetadataIndexTemplateService.java | 2 +- .../common/io/stream/DelayableWriteable.java | 6 +- .../elasticsearch/common/lucene/Lucene.java | 27 +++--- .../org/elasticsearch/index/IndexService.java | 26 +++++- .../index/query/QueryShardContext.java | 37 +++++++- .../search/DefaultSearchContext.java | 3 +- .../elasticsearch/search/SearchService.java | 4 +- .../search/internal/ShardSearchRequest.java | 27 +++--- .../search/slice/SliceBuilder.java | 4 +- .../search/sort/FieldSortBuilder.java | 16 +++- .../search/sort/ShardDocSortField.java | 85 ++++++++++++++++++ .../search/sort/SortBuilders.java | 8 ++ .../MetadataCreateIndexServiceTests.java | 2 +- .../common/lucene/LuceneTests.java | 8 +- .../fielddata/AbstractFieldDataTestCase.java | 2 +- .../index/mapper/DateFieldTypeTests.java | 6 +- .../mapper/FieldNamesFieldTypeTests.java | 2 +- .../index/mapper/IndexFieldTypeTests.java | 2 +- .../index/mapper/NumberFieldTypeTests.java | 2 +- .../index/mapper/RangeFieldTypeTests.java | 2 +- .../index/query/QueryShardContextTests.java | 3 +- .../index/query/RangeQueryRewriteTests.java | 6 +- .../index/search/MultiMatchQueryTests.java | 69 ++++++++++++--- .../index/search/NestedHelperTests.java | 9 +- .../search/nested/NestedSortingTests.java | 2 +- .../search/DefaultSearchContextTests.java | 4 +- .../fetch/subphase/FieldFetcherTests.java | 2 +- .../highlight/HighlightBuilderTests.java | 2 +- .../rescore/QueryRescorerBuilderTests.java | 4 +- .../search/sort/AbstractSortTestCase.java | 2 +- .../search/sort/FieldSortBuilderTests.java | 17 ++++ .../search/sort/SortBuilderTests.java | 13 ++- .../AbstractSuggestionBuilderTestCase.java | 4 +- .../aggregations/AggregatorTestCase.java | 12 +++ .../test/AbstractBuilderTestCase.java | 2 +- .../elasticsearch/test/TestSearchContext.java | 2 +- .../xpack/core/search/PointInTimeIT.java | 86 +++++++++++++++++++ .../DocumentSubsetBitsetCacheTests.java | 2 +- ...ityIndexReaderWrapperIntegrationTests.java | 4 +- .../action/EnrichShardMultiSearchAction.java | 1 + .../job/RollupIndexerIndexingTests.java | 2 +- .../xpack/security/Security.java | 1 + .../mapper/WildcardFieldMapperTests.java | 2 +- 54 files changed, 489 insertions(+), 104 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index c7ac35d3febce..b4d5ddc55d0ed 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -85,7 +85,7 @@ public void setup() { indexService = createIndex("test", settings, "t", "text_shingle", "type=text,analyzer=text_shingle", "text_shingle_unigram", "type=text,analyzer=text_shingle_unigram"); - shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); + shardContext = indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap()); // parsed queries for "text_shingle_unigram:(foo bar baz)" with query parsers // that ignores position length attribute diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 1810a1eff384a..4b6109f63df7b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -556,7 +556,7 @@ private static Response prepareRamIndex(Request request, searcher.setQueryCache(null); final long absoluteStartMillis = System.currentTimeMillis(); QueryShardContext context = - indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null, emptyMap()); + indexService.newQueryShardContext(0, 0, searcher, () -> absoluteStartMillis, null, emptyMap()); return handler.apply(context, indexReader.leaves().get(0)); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index ca885cdfdff6a..5630973d66448 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -47,7 +47,7 @@ public void testNeedsScores() { contexts.put(NumberSortScript.CONTEXT, Whitelist.BASE_WHITELISTS); PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); - QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null, emptyMap()); + QueryShardContext shardContext = index.newQueryShardContext(0, 0, null, () -> 0, null, emptyMap()); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 45a836562913a..9da5505623bb1 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -514,7 +514,7 @@ public void testQueryWithRewrite() throws Exception { XContentType.JSON)); BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); QueryShardContext shardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { + randomInt(20), 0, null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index ff5d6dc7c4b7b..c9414059bed6e 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -259,7 +259,7 @@ public void testRangeQueriesWithNow() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { long[] currentTime = new long[] {System.currentTimeMillis()}; QueryShardContext queryShardContext = - indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null, emptyMap()); + indexService.newQueryShardContext(0, 0, searcher, () -> currentTime[0], null, emptyMap()); BytesReference source = BytesReference.bytes(jsonBuilder().startObject() .field("field1", "value") diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index 36ac1f0c7548f..41c8ef3500a85 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -156,8 +156,8 @@ --- "date_nanos": - skip: - version: " - 7.99.99" - reason: fixed in 8.0.0 to be backported to 7.10.0 + version: " - 7.9.99" + reason: fixed in 7.10.0 - do: indices.create: @@ -218,3 +218,27 @@ - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" } - match: {hits.hits.0.sort: [1571617804828740000] } + +--- +"_shard_doc sort": + - skip: + version: " - 7.99.99" + reason: _shard_doc sort was added in 8.0 (TODO adapt version after backport) + + - do: + indices.create: + index: test + - do: + index: + index: test + id: 1 + body: { id: 1, foo: bar, age: 18 } + + - do: + catch: /\[_shard_doc\] sort field cannot be used without \[point in time\]/ + search: + index: test + body: + size: 1 + sort: [{ _shard_doc }] + search_after: [ 0L ] diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index db129a9479934..2ee2684530ae3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -183,7 +183,7 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()))); + tempIndexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap()))); Map aliasesByName = aliases.stream().collect( Collectors.toMap(AliasMetadata::getAlias, Function.identity())); diff --git a/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java index a84d5885a73c3..687adde514839 100644 --- a/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java +++ b/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java @@ -290,9 +290,13 @@ long ramBytesUsedQueryResult(QuerySearchResult result) { if (hasAggs == false) { return 0; } - return result.aggregations() - .asSerialized(InternalAggregations::readFrom, namedWriteableRegistry) - .ramBytesUsed(); + if (result.aggregations().isSerialized()) { + return result.aggregations() + .asSerialized(InternalAggregations::readFrom, namedWriteableRegistry) + .ramBytesUsed(); + } else { + return result.aggregations().expand().getSerializedSize(); + } } /** diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index e6e07dbc67904..1b8275af4ade0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -34,6 +34,9 @@ import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.tasks.TaskId; import java.io.IOException; @@ -280,6 +283,14 @@ public ActionRequestValidationException validate() { if (scroll) { validationException = addValidationError("using [point in time] is not allowed in a scroll context", validationException); } + } else if (source != null && source.sorts() != null) { + for (SortBuilder sortBuilder : source.sorts()) { + if (sortBuilder instanceof FieldSortBuilder + && ShardDocSortField.NAME.equals(((FieldSortBuilder) sortBuilder).getFieldName())) { + validationException = addValidationError("[" + FieldSortBuilder.SHARD_DOC_FIELD_NAME + + "] sort field cannot be used without [point in time]", validationException); + } + } } return validationException; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 0b5210baabfd2..fec1f15962d24 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -487,7 +487,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), + xContentRegistry, indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), metadataTransformer); } @@ -520,7 +520,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), + xContentRegistry, indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())), Collections.singletonList(templateName), metadataTransformer); } @@ -566,7 +566,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt currentState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), + indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())), List.of(), metadataTransformer); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 3ced1267655f1..605c569a86dbf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -149,8 +149,8 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable System.currentTimeMillis(), null, emptyMap()), xContentRegistry); + aliasValidator.validateAliasFilter(alias, filter, indexService.newQueryShardContext(0, -1, + null, () -> System.currentTimeMillis(), null, emptyMap()), xContentRegistry); } }; if (action.apply(newAliasValidator, metadata, index)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index ddab81145eff3..3f40294737206 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1118,7 +1118,7 @@ private static void validateCompositeTemplate(final ClusterState state, new AliasValidator(), // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())); + xContentRegistry, tempIndexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())); // triggers inclusion of _timestamp field and its validation: String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java index 1e58f12985c17..1ced33ce6ed7f 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java @@ -78,7 +78,7 @@ private DelayableWriteable() {} * {@code true} if the {@linkplain Writeable} is being stored in * serialized form, {@code false} otherwise. */ - abstract boolean isSerialized(); + public abstract boolean isSerialized(); private static class Referencing extends DelayableWriteable { private final T reference; @@ -109,7 +109,7 @@ public Serialized asSerialized(Reader reader, NamedWriteableRegistry regis } @Override - boolean isSerialized() { + public boolean isSerialized() { return false; } @@ -179,7 +179,7 @@ public Serialized asSerialized(Reader reader, NamedWriteableRegistry regis } @Override - boolean isSerialized() { + public boolean isSerialized() { return true; } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index b9f8bb98b4273..865bb793b4438 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -93,6 +93,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.search.sort.ShardDocSortField; import java.io.IOException; import java.math.BigInteger; @@ -566,29 +567,35 @@ public static void writeSortType(StreamOutput out, SortField.Type sortType) thro out.writeVInt(sortType.ordinal()); } - public static void writeSortField(StreamOutput out, SortField sortField) throws IOException { + /** + * Returns the generic version of the provided {@link SortField} that + * can be used to merge documents coming from different shards. + */ + private static SortField rewriteMergeSortField(SortField sortField) { if (sortField.getClass() == GEO_DISTANCE_SORT_TYPE_CLASS) { - // for geo sorting, we replace the SortField with a SortField that assumes a double field. - // this works since the SortField is only used for merging top docs SortField newSortField = new SortField(sortField.getField(), SortField.Type.DOUBLE); newSortField.setMissingValue(sortField.getMissingValue()); - sortField = newSortField; + return newSortField; } else if (sortField.getClass() == SortedSetSortField.class) { - // for multi-valued sort field, we replace the SortedSetSortField with a simple SortField. - // It works because the sort field is only used to merge results from different shards. SortField newSortField = new SortField(sortField.getField(), SortField.Type.STRING, sortField.getReverse()); newSortField.setMissingValue(sortField.getMissingValue()); - sortField = newSortField; + return newSortField; } else if (sortField.getClass() == SortedNumericSortField.class) { - // for multi-valued sort field, we replace the SortedSetSortField with a simple SortField. - // It works because the sort field is only used to merge results from different shards. SortField newSortField = new SortField(sortField.getField(), ((SortedNumericSortField) sortField).getNumericType(), sortField.getReverse()); newSortField.setMissingValue(sortField.getMissingValue()); - sortField = newSortField; + return sortField; + } else if (sortField.getClass() == ShardDocSortField.class) { + SortField newSortField = new SortField(sortField.getField(), SortField.Type.LONG, sortField.getReverse()); + return newSortField; + } else { + return sortField; } + } + public static void writeSortField(StreamOutput out, SortField sortField) throws IOException { + sortField = rewriteMergeSortField(sortField); if (sortField.getClass() != SortField.class) { throw new IllegalArgumentException("Cannot serialize SortField impl [" + sortField + "]"); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index bec23def8b1c3..18c8bfa5d77c0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -192,7 +192,7 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); + () -> newQueryShardContext(0, 0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. @@ -588,6 +588,7 @@ public IndexSettings getIndexSettings() { */ public QueryShardContext newQueryShardContext( int shardId, + int shardRequestIndex, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias, @@ -596,9 +597,26 @@ public QueryShardContext newQueryShardContext( final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( - shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), - similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); + shardId, + shardRequestIndex, + indexSettings, + bigArrays, + indexCache.bitsetFilterCache(), + indexFieldData::getForField, + mapperService(), + similarityService(), + scriptService, + xContentRegistry, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + clusterAlias, + indexNameMatcher, + allowExpensiveQueries, + valuesSourceRegistry, + runtimeMappings + ); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index d6a358f5aad1d..2df7b86f36dce 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -94,6 +94,7 @@ public class QueryShardContext extends QueryRewriteContext { private final BitsetFilterCache bitsetFilterCache; private final TriFunction, IndexFieldData> indexFieldDataService; private final int shardId; + private final int shardRequestIndex; private final IndexSearcher searcher; private boolean cacheable = true; private final SetOnce frozen = new SetOnce<>(); @@ -114,6 +115,7 @@ public class QueryShardContext extends QueryRewriteContext { */ public QueryShardContext( int shardId, + int shardRequestIndex, IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, @@ -134,6 +136,7 @@ public QueryShardContext( ) { this( shardId, + shardRequestIndex, indexSettings, bigArrays, bitsetFilterCache, @@ -158,13 +161,30 @@ public QueryShardContext( } public QueryShardContext(QueryShardContext source) { - this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, - source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), - source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, source.runtimeMappings); + this( + source.shardId, + source.shardRequestIndex, + source.indexSettings, + source.bigArrays, + source.bitsetFilterCache, + source.indexFieldDataService, + source.mapperService, + source.similarityService, + source.scriptService, + source.getXContentRegistry(), + source.getWriteableRegistry(), + source.client, source.searcher, + source.nowInMillis, + source.indexNameMatcher, + source.fullyQualifiedIndex, + source.allowExpensiveQueries, + source.valuesSourceRegistry, + source.runtimeMappings + ); } private QueryShardContext(int shardId, + int shardRequestIndex, IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, @@ -184,6 +204,7 @@ private QueryShardContext(int shardId, Map runtimeMappings) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; + this.shardRequestIndex = shardRequestIndex; this.similarityService = similarityService; this.mapperService = mapperService; this.bigArrays = bigArrays; @@ -520,6 +541,14 @@ public int getShardId() { return shardId; } + /** + * Returns the shard request ordinal that is used by the main search request + * to reference this shard. + */ + public int getShardRequestIndex() { + return shardRequestIndex; + } + @Override public final long nowInMillis() { failIfFrozen(); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 6184b18048be9..a04607d5752ac 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -166,7 +166,8 @@ final class DefaultSearchContext extends SearchContext { this.timeout = timeout; queryShardContext = indexService.newQueryShardContext( request.shardId().id(), - this.searcher, + request.shardRequestIndex(), + searcher, request::nowInMillis, shardTarget.getClusterAlias(), request.getRuntimeMappings() diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 8c416d9b0912f..90f8dc4cbbd15 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1174,8 +1174,8 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre } try (canMatchSearcher) { - QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher, - request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); + QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), 0, + canMatchSearcher, request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); Rewriteable.rewrite(request.getRewriteable(), context, false); final boolean aliasFilterCanMatch = request.getAliasFilter() .getQueryBuilder() instanceof MatchNoneQueryBuilder == false; diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 1d768ab8c7b63..7a3a6a145ea04 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -72,7 +72,7 @@ public class ShardSearchRequest extends TransportRequest implements IndicesRequest { private final String clusterAlias; private final ShardId shardId; - private final int shardIndex; + private final int shardRequestIndex; private final int numberOfShards; private final SearchType searchType; private final Scroll scroll; @@ -94,20 +94,20 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque public ShardSearchRequest(OriginalIndices originalIndices, SearchRequest searchRequest, ShardId shardId, - int shardIndex, + int shardRequestIndex, int numberOfShards, AliasFilter aliasFilter, float indexBoost, long nowInMillis, @Nullable String clusterAlias) { - this(originalIndices, searchRequest, shardId, shardIndex, numberOfShards, aliasFilter, + this(originalIndices, searchRequest, shardId, shardRequestIndex, numberOfShards, aliasFilter, indexBoost, nowInMillis, clusterAlias, null, null); } public ShardSearchRequest(OriginalIndices originalIndices, SearchRequest searchRequest, ShardId shardId, - int shardIndex, + int shardRequestIndex, int numberOfShards, AliasFilter aliasFilter, float indexBoost, @@ -117,7 +117,7 @@ public ShardSearchRequest(OriginalIndices originalIndices, TimeValue keepAlive) { this(originalIndices, shardId, - shardIndex, + shardRequestIndex, numberOfShards, searchRequest.searchType(), searchRequest.source(), @@ -144,7 +144,7 @@ public ShardSearchRequest(ShardId shardId, private ShardSearchRequest(OriginalIndices originalIndices, ShardId shardId, - int shardIndex, + int shardRequestIndex, int numberOfShards, SearchType searchType, SearchSourceBuilder source, @@ -158,7 +158,7 @@ private ShardSearchRequest(OriginalIndices originalIndices, ShardSearchContextId readerId, TimeValue keepAlive) { this.shardId = shardId; - this.shardIndex = shardIndex; + this.shardRequestIndex = shardRequestIndex; this.numberOfShards = numberOfShards; this.searchType = searchType; this.source = source; @@ -179,7 +179,7 @@ public ShardSearchRequest(StreamInput in) throws IOException { super(in); shardId = new ShardId(in); searchType = SearchType.fromId(in.readByte()); - shardIndex = in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readVInt() : -1; + shardRequestIndex = in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readVInt() : -1; numberOfShards = in.readVInt(); scroll = in.readOptionalWriteable(Scroll::new); source = in.readOptionalWriteable(SearchSourceBuilder::new); @@ -218,7 +218,7 @@ public ShardSearchRequest(StreamInput in) throws IOException { public ShardSearchRequest(ShardSearchRequest clone) { this.shardId = clone.shardId; - this.shardIndex = clone.shardIndex; + this.shardRequestIndex = clone.shardRequestIndex; this.searchType = clone.searchType; this.numberOfShards = clone.numberOfShards; this.scroll = clone.scroll; @@ -248,7 +248,7 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce out.writeByte(searchType.id()); if (asKey == false) { if (out.getVersion().onOrAfter(Version.V_7_11_0)) { - out.writeVInt(shardIndex); + out.writeVInt(shardRequestIndex); } out.writeVInt(numberOfShards); } @@ -315,10 +315,11 @@ public void source(SearchSourceBuilder source) { } /** - * Returns the index of the shard that is used to tiebreak documents with identical sort values. + * Returns the shard request ordinal that is used by the main search request + * to reference this shard. */ - public int shardIndex() { - return shardIndex; + public int shardRequestIndex() { + return shardRequestIndex; } public int numberOfShards() { diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java index 29e95e4b7d680..a6d25efece134 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java @@ -203,8 +203,8 @@ public Query toFilter(ShardSearchRequest request, QueryShardContext context) { throw new IllegalArgumentException("field " + field + " not found"); } - int shardIndex = request.shardIndex() != -1 ? request.shardIndex() : request.shardId().id(); - int numShards = request.shardIndex() != -1 ? request.numberOfShards() : context.getIndexSettings().getNumberOfShards(); + int shardIndex = request.shardRequestIndex() != -1 ? request.shardRequestIndex() : request.shardId().id(); + int numShards = request.shardRequestIndex() != -1 ? request.numberOfShards() : context.getIndexSettings().getNumberOfShards(); String field = this.field; boolean useTermQuery = false; if (IdFieldMapper.NAME.equals(field)) { diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index 7f197d3398038..d7eb3d5aaf024 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -81,6 +81,12 @@ public class FieldSortBuilder extends SortBuilder { * special field name to sort by index order */ public static final String DOC_FIELD_NAME = "_doc"; + + /** + * special field name to sort by index order + */ + public static final String SHARD_DOC_FIELD_NAME = ShardDocSortField.NAME; + private static final SortFieldAndFormat SORT_DOC = new SortFieldAndFormat( new SortField(null, SortField.Type.DOC), DocValueFormat.RAW); private static final SortFieldAndFormat SORT_DOC_REVERSE = new SortFieldAndFormat( @@ -321,8 +327,15 @@ private static NumericType resolveNumericType(String value) { @Override public SortFieldAndFormat build(QueryShardContext context) throws IOException { + final boolean reverse = order == SortOrder.DESC; + if (DOC_FIELD_NAME.equals(fieldName)) { - return order == SortOrder.DESC ? SORT_DOC_REVERSE : SORT_DOC; + return reverse ? SORT_DOC_REVERSE : SORT_DOC; + } else if (SHARD_DOC_FIELD_NAME.equals(fieldName)) { + if (context.getShardRequestIndex() == -1) { + throw new IllegalArgumentException(""); + } + return new SortFieldAndFormat(new ShardDocSortField(context.getShardRequestIndex(), reverse), DocValueFormat.RAW); } MappedFieldType fieldType = context.getFieldType(fieldName); @@ -331,7 +344,6 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { fieldType = resolveUnmappedType(context); } - boolean reverse = order == SortOrder.DESC; IndexFieldData fieldData = context.getForField(fieldType); if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { diff --git a/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java new file mode 100644 index 0000000000000..47907875d8a8d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.sort; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.comparators.DocComparator; + + /** + * A {@link SortField} that first compares the shard index and then uses the document number (_doc) + * to tiebreak if the value is the same. + **/ +public class ShardDocSortField extends SortField { + public static final String NAME = "_shard_doc"; + + private final int shardRequestIndex; + + public ShardDocSortField(int shardRequestIndex, boolean reverse) { + super(NAME, Type.LONG, reverse); + assert shardRequestIndex >= 0; + this.shardRequestIndex = shardRequestIndex; + } + + int getShardRequestIndex() { + return shardRequestIndex; + } + + @Override + public FieldComparator getComparator(int numHits, int sortPos) { + final DocComparator delegate = new DocComparator(numHits, false, sortPos); + + return new FieldComparator() { + @Override + public int compare(int slot1, int slot2) { + return delegate.compare(slot1, slot2); + } + + @Override + public int compareValues(Long first, Long second) { + return Long.compare(first, second); + } + + @Override + public void setTopValue(Long value) { + int topShardIndex = (int) (value >> 32); + if (shardRequestIndex == topShardIndex) { + delegate.setTopValue(value.intValue()); + } else if (shardRequestIndex < topShardIndex) { + delegate.setTopValue(Integer.MAX_VALUE); + } else { + delegate.setTopValue(-1); + } + } + + @Override + public Long value(int slot) { + return (((long) shardRequestIndex) << 32) | (delegate.value(slot) & 0xFFFFFFFFL); + } + + @Override + public LeafFieldComparator getLeafComparator(LeafReaderContext context) { + return delegate.getLeafComparator(context); + } + }; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java b/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java index 3eae9b8d01960..7f5d98dee82ec 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.script.Script; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; /** @@ -46,6 +47,13 @@ public static FieldSortBuilder fieldSort(String field) { return new FieldSortBuilder(field); } + /** + * Constructs a sort tiebreaker that can be used within a point in time reader {@link PointInTimeBuilder}. + */ + public static FieldSortBuilder pitTiebreaker() { + return new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME); + } + /** * Constructs a new script based sort. * diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 4c5f8c08abedc..376de20cabda2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -123,7 +123,7 @@ public void setupCreateIndexRequestAndAliasValidator() { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); Settings indexSettings = Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); - queryShardContext = new QueryShardContext(0, + queryShardContext = new QueryShardContext(0, 0, new IndexSettings(IndexMetadata.builder("test").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index a716350638d9d..2da356426e43e 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -68,6 +68,7 @@ import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -716,7 +717,7 @@ private static Tuple randomSortFieldCustomComparatorSource private static Tuple randomCustomSortField() { String field = randomAlphaOfLengthBetween(3, 10); - switch(randomIntBetween(0, 2)) { + switch(randomIntBetween(0, 3)) { case 0: { SortField sortField = LatLonDocValuesField.newDistanceSort(field, 0, 0); SortField expected = new SortField(field, SortField.Type.DOUBLE); @@ -742,6 +743,11 @@ private static Tuple randomCustomSortField() { } return Tuple.tuple(sortField, expected); } + case 3: { + ShardDocSortField sortField = new ShardDocSortField(randomIntBetween(0, 100), randomBoolean()); + SortField expected = new SortField(ShardDocSortField.NAME, SortField.Type.LONG, sortField.getReverse()); + return Tuple.tuple(sortField, expected); + } default: throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 0438577b219c6..292d7fb034917 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -139,7 +139,7 @@ public void setup() throws Exception { writer = new IndexWriter( new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()) ); - shardContext = indexService.newQueryShardContext(0, null, () -> 0, null, emptyMap()); + shardContext = indexService.newQueryShardContext(0, 0, null, () -> 0, null, emptyMap()); } protected final List refreshReader() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index d7ce0a3460e23..06d42b41af0ad 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -165,7 +165,7 @@ public void testValueForSearch() { public void testTermQuery() { Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); - QueryShardContext context = new QueryShardContext(0, + QueryShardContext context = new QueryShardContext(0, 0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); @@ -187,7 +187,7 @@ public void testTermQuery() { public void testRangeQuery() throws IOException { Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); - QueryShardContext context = new QueryShardContext(0, + QueryShardContext context = new QueryShardContext(0, 0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); @@ -232,7 +232,7 @@ public void testRangeQueryWithIndexSort() { .build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); - QueryShardContext context = new QueryShardContext(0, indexSettings, + QueryShardContext context = new QueryShardContext(0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null, emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index 7e93b0e79c63c..c78346f469439 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -50,7 +50,7 @@ public void testTermQuery() { when(mapperService.fieldType("field_name")).thenReturn(fieldType); when(mapperService.simpleMatchToFullName("field_name")).thenReturn(Collections.singleton("field_name")); - QueryShardContext queryShardContext = new QueryShardContext(0, + QueryShardContext queryShardContext = new QueryShardContext(0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, null, null, null, null, () -> 0L, null, null, () -> true, null, emptyMap()); Query termQuery = fieldNamesFieldType.termQuery("field_name", queryShardContext); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index b858929e3abc4..cd24c8bb48d11 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -69,7 +69,7 @@ private QueryShardContext createContext() { IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); Predicate indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index"); - return new QueryShardContext(0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), + return new QueryShardContext(0, 0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, System::currentTimeMillis, null, indexNameMatcher, () -> true, null, emptyMap()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index ac763d35a0c3f..397823d8347a7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -473,7 +473,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS DirectoryReader reader = DirectoryReader.open(w); IndexSearcher searcher = newSearcher(reader); - QueryShardContext context = new QueryShardContext(0, indexSettings, + QueryShardContext context = new QueryShardContext(0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null, emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index a4c026f6d9799..7c4465c03c1d1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -213,7 +213,7 @@ private QueryShardContext createContext() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); } diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 4bd5bce9f479b..47f955fc4a48a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -195,6 +195,7 @@ public void testIndexSortedOnField() { IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); QueryShardContext context = new QueryShardContext( + 0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, @@ -374,7 +375,7 @@ private static QueryShardContext createQueryShardContext( MapperService mapperService = createMapperService(indexUuid, fieldTypeLookup, mapperPlugins); final long nowInMillis = randomNonNegativeLong(); return new QueryShardContext( - 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, + 0, 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), null, null, () -> nowInMillis, clusterAlias, null, () -> true, null, runtimeMappings); diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index 9ab1e80f09bdc..d7f6cb3a6c61d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -41,7 +41,7 @@ public class RangeQueryRewriteTests extends ESSingleNodeTestCase { public void testRewriteMissingField() throws Exception { IndexService indexService = createIndex("test"); IndexReader reader = new MultiReader(); - QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, + QueryRewriteContext context = new QueryShardContext(0, 0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, new IndexSearcher(reader), null, null, null, () -> true, null, emptyMap()); RangeQueryBuilder range = new RangeQueryBuilder("foo"); @@ -59,7 +59,7 @@ public void testRewriteMissingReader() throws Exception { .endObject().endObject()); indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); - QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, null, + QueryRewriteContext context = new QueryShardContext(0, 0, indexService.getIndexSettings(), null, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, null, null, null, null, () -> true, null, emptyMap()); RangeQueryBuilder range = new RangeQueryBuilder("foo"); @@ -79,7 +79,7 @@ public void testRewriteEmptyReader() throws Exception { indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); IndexReader reader = new MultiReader(); - QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, + QueryRewriteContext context = new QueryShardContext(0, 0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, new IndexSearcher(reader), null, null, null, () -> true, null, emptyMap()); RangeQueryBuilder range = new RangeQueryBuilder("foo"); diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index d633b7248e2cb..840a20ccf82ad 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -100,7 +100,13 @@ public void setup() throws IOException { public void testCrossFieldMultiMatchQuery() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); queryShardContext.setAllowUnmappedFields(true); for (float tieBreaker : new float[] {0.0f, 0.5f}) { Query parsedQuery = multiMatchQuery("banon") @@ -127,7 +133,14 @@ public void testBlendTerms() { float[] boosts = new float[] {2, 3}; Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), 1f, false, @@ -151,7 +164,14 @@ public Query termQuery(Object value, QueryShardContext context) { BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f) ), 1f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), 1f, true, @@ -168,7 +188,14 @@ public Query termQuery(Object value, QueryShardContext context) { } }; expectThrows(IllegalArgumentException.class, () -> MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft, 1)))); } @@ -190,7 +217,14 @@ public Query termQuery(Object value, QueryShardContext context) { expectedDisjunct1 ), 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), 1f, false, @@ -201,7 +235,13 @@ public Query termQuery(Object value, QueryShardContext context) { public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); @@ -233,7 +273,13 @@ public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { public void testMultiMatchCrossFieldsWithSynonymsPhrase() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); Map fieldNames = new HashMap<>(); @@ -300,9 +346,12 @@ public void testKeywordSplitQueriesOnWhitespace() throws IOException { .endObject().endObject()); mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { - throw new UnsupportedOperationException(); - }, null, emptyMap()); + randomInt(20), + 0, + null, () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); Map fieldNames = new HashMap<>(); fieldNames.put("field", 1.0f); diff --git a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java index d8f7405ef3387..6386d90d0a788 100644 --- a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java @@ -335,7 +335,14 @@ public void testConjunction() { } public void testNested() throws IOException { - QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null, emptyMap()); + QueryShardContext context = indexService.newQueryShardContext( + 0, + 0, + new IndexSearcher(new MultiReader()), + () -> 0, + null, + emptyMap() + ); NestedQueryBuilder queryBuilder = new NestedQueryBuilder("nested1", new MatchAllQueryBuilder(), ScoreMode.Avg); ESToParentBlockJoinQuery query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java index e4b0573b06cca..cd7399ea3256d 100644 --- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java @@ -608,7 +608,7 @@ public void testMultiLevelNestedSorting() throws IOException { DirectoryReader reader = DirectoryReader.open(writer); reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(reader); - QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null, emptyMap()); + QueryShardContext queryShardContext = indexService.newQueryShardContext(0, 0, searcher, () -> 0L, null, emptyMap()); FieldSortBuilder sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count"); sortBuilder.setNestedSort(new NestedSortBuilder("chapters").setNestedSort(new NestedSortBuilder("chapters.paragraphs"))); diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index e4384bd344a36..b426df9e21895 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -79,7 +79,7 @@ public void testPreProcess() throws Exception { when(shardSearchRequest.searchType()).thenReturn(SearchType.DEFAULT); ShardId shardId = new ShardId("index", UUID.randomUUID().toString(), 1); when(shardSearchRequest.shardId()).thenReturn(shardId); - when(shardSearchRequest.shardIndex()).thenReturn(shardId.id()); + when(shardSearchRequest.shardRequestIndex()).thenReturn(shardId.id()); when(shardSearchRequest.numberOfShards()).thenReturn(2); ThreadPool threadPool = new TestThreadPool(this.getClass().getName()); @@ -106,7 +106,7 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())).thenReturn( + when(indexService.newQueryShardContext(eq(shardId.id()), 0, anyObject(), anyObject(), anyString(), anyObject())).thenReturn( queryShardContext ); MapperService mapperService = mock(MapperService.class); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 00d17464f4e70..49193f482d831 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -710,7 +710,7 @@ private static QueryShardContext newQueryShardContext(MapperService mapperServic .put(IndexMetadata.SETTING_INDEX_UUID, "uuid").build(); IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); - return new QueryShardContext(0, indexSettings, null, null, null, mapperService, null, null, null, null, null, null, null, null, + return new QueryShardContext(0, 0, indexSettings, null, null, null, mapperService, null, null, null, null, null, null, null, null, null, null, null, emptyMap()); } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index f30158f2fbad0..6bf9fb5ee116f 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -278,7 +278,7 @@ public void testBuildSearchContextHighlight() throws IOException { Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_"); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null, emptyMap()) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 4c2749c5c0c7b..6e04efbd5d170 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -142,7 +142,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()) { @Override @@ -186,7 +186,7 @@ public void testRewritingKeepsSettings() throws IOException { .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 38a721fd117dc..e9cb1ce7aff37 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -201,7 +201,7 @@ protected final QueryShardContext createMockShardContext(IndexSearcher searcher) IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 89d6aa235f1df..c30fb30306753 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -72,6 +72,7 @@ import static org.elasticsearch.search.sort.FieldSortBuilder.getMinMaxOrNull; import static org.elasticsearch.search.sort.FieldSortBuilder.getPrimaryFieldSortOrNull; import static org.elasticsearch.search.sort.NestedSortBuilderTests.createRandomNestedSort; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class FieldSortBuilderTests extends AbstractSortTestCase { @@ -160,6 +161,8 @@ protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField SortField.Type expectedType; if (builder.getFieldName().equals(FieldSortBuilder.DOC_FIELD_NAME)) { expectedType = SortField.Type.DOC; + } else if (builder.getFieldName().equals(FieldSortBuilder.SHARD_DOC_FIELD_NAME)) { + expectedType = SortField.Type.LONG; } else { expectedType = SortField.Type.CUSTOM; } @@ -324,6 +327,20 @@ public void testUnknownOptionFails() throws IOException { } } + public void testShardDocSort() throws IOException { + QueryShardContext shardContextMock = createMockShardContext(); + + boolean reverse = randomBoolean(); + FieldSortBuilder sortBuilder = new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME) + .order(reverse ? SortOrder.DESC : SortOrder.ASC); + SortFieldAndFormat sortAndFormat = sortBuilder.build(shardContextMock); + assertThat(sortAndFormat.field.getClass(), equalTo(ShardDocSortField.class)); + ShardDocSortField sortField = (ShardDocSortField) sortAndFormat.field; + assertThat(sortField.getShardRequestIndex(), equalTo(shardContextMock.getShardRequestIndex())); + assertThat(sortField.getReverse(), equalTo(reverse)); + assertThat(sortAndFormat.format, equalTo(DocValueFormat.RAW)); + } + @Override protected MappedFieldType provideMappedFieldType(String name) { if (name.equals(MAPPED_STRING_FIELDNAME)) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java index 7b1e56d5c69f2..6c27c23b78573 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java @@ -190,20 +190,25 @@ public static List> randomSortBuilderList() { int size = randomIntBetween(1, 5); List> list = new ArrayList<>(size); for (int i = 0; i < size; i++) { - switch (randomIntBetween(0, 3)) { + switch (randomIntBetween(0, 5)) { case 0: list.add(new ScoreSortBuilder()); break; case 1: - String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAlphaOfLengthBetween(1, 10); - list.add(new FieldSortBuilder(fieldName)); + list.add(new FieldSortBuilder( randomAlphaOfLengthBetween(1, 10))); break; case 2: - list.add(GeoDistanceSortBuilderTests.randomGeoDistanceSortBuilder()); + list.add(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME)); break; case 3: + list.add(GeoDistanceSortBuilderTests.randomGeoDistanceSortBuilder()); + break; + case 4: list.add(ScriptSortBuilderTests.randomScriptSortBuilder()); break; + case 5: + list.add(SortBuilders.pitTiebreaker()); + break; default: throw new IllegalStateException("unexpected randomization in tests"); } diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index f9892be86028a..3ac6396e580d0 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -179,7 +179,7 @@ public NamedAnalyzer get(Object key) { when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); when(scriptService.compile(any(Script.class), any())).then(invocation -> new TestTemplateService.MockTemplateScript.Factory( ((Script) invocation.getArguments()[0]).getIdOrCode())); - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null, emptyMap()); @@ -219,7 +219,7 @@ public void testBuildWithUnmappedField() { when(mapperService.getNamedAnalyzer(any(String.class))).then( invocation -> new NamedAnalyzer((String) invocation.getArguments()[0], AnalyzerScope.INDEX, new SimpleAnalyzer())); - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null, emptyMap()); if (randomBoolean()) { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 44f982df2beae..bca84b1c3cf67 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -271,6 +271,7 @@ protected AggregationContext createAggregationContext(IndexSearcher indexSearche searchLookup) -> fieldType.fielddataBuilder(mapperService.getIndexSettings().getIndex().getName(), searchLookup) .build(new IndexFieldDataCache.None(), breakerService); QueryShardContext queryShardContext = new QueryShardContext( + 0, 0, indexSettings, bigArrays, @@ -370,6 +371,17 @@ protected MapperService mapperServiceMock() { } /** + * Sub-tests that need a more complex index field data provider can override this + */ + protected TriFunction, IndexFieldData> getIndexFieldDataLookup( + MapperService mapperService, CircuitBreakerService circuitBreakerService) { + return (fieldType, s, searchLookup) -> fieldType.fielddataBuilder( + mapperService.getIndexSettings().getIndex().getName(), searchLookup) + .build(new IndexFieldDataCache.None(), circuitBreakerService); + } + + /** +>>>>>>> Stashed changes * Sub-tests that need scripting can override this method to provide a script service and pre-baked scripts */ protected ScriptService getMockScriptService() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index f44cc47924576..b66156c28cc76 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -411,7 +411,7 @@ public void close() throws IOException { } QueryShardContext createShardContext(IndexSearcher searcher) { - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataService::getForField, mapperService, similarityService, scriptService, xContentRegistry, namedWriteableRegistry, this.client, searcher, () -> nowInMillis, null, indexNameMatcher(), () -> true, null, emptyMap()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 1a15be52bb1ba..4533a18f391a3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -96,7 +96,7 @@ public TestSearchContext(IndexService indexService) { this.indexService = indexService; this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.indexShard = indexService.getShardOrNull(0); - queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); + queryShardContext = indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap()); } public TestSearchContext(QueryShardContext queryShardContext) { diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java index e39632966b440..75679a437de10 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java @@ -9,6 +9,8 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.search.ShardSearchFailure; @@ -25,8 +27,12 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchContextMissingException; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.search.action.ClosePointInTimeAction; @@ -37,6 +43,7 @@ import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -370,6 +377,85 @@ public void testPartialResults() throws Exception { } } + public void testPITTiebreak() throws Exception { + assertAcked(client().admin().indices().prepareDelete("index-*").get()); + int numIndex = randomIntBetween(2, 10); + int expectedNumDocs = 0; + for (int i = 0; i < numIndex; i++) { + String index = "index-" + i; + createIndex(index, Settings.builder().put("index.number_of_shards", 1).build()); + int numDocs = randomIntBetween(3, 20); + for (int j = 0; j < numDocs; j++) { + client().prepareIndex(index).setSource("value", randomIntBetween(0, 2)).get(); + expectedNumDocs ++; + } + } + refresh("index-*"); + String pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueHours(1)); + try { + for (int size = 1; size <= numIndex; size++) { + SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC; + assertPagination(new PointInTimeBuilder(pit), expectedNumDocs, size, + SortBuilders.pitTiebreaker().order(order)); + assertPagination(new PointInTimeBuilder(pit), expectedNumDocs, size, + SortBuilders.fieldSort("value"), SortBuilders.pitTiebreaker().order(order)); + } + } finally { + closePointInTime(pit); + } + } + + private void assertPagination(PointInTimeBuilder pit, int expectedNumDocs, int size, SortBuilder... sort) throws Exception { + Set seen = new HashSet<>(); + SearchRequestBuilder builder = client().prepareSearch() + .setSize(size) + .setPointInTime(pit); + + final int[] reverseMuls = new int[sort.length]; + for (int i = 0; i < sort.length; i++) { + builder.addSort(sort[i]); + reverseMuls[i] = sort[i].order() == SortOrder.ASC ? 1 : -1; + } + final SearchRequest searchRequest = builder.request(); + SearchResponse response = client().search(searchRequest).get(); + Object[] lastSortValues = null; + while (response.getHits().getHits().length > 0) { + Object[] lastHitSortValues = null; + for (SearchHit hit : response.getHits().getHits()) { + assertTrue(seen.add(hit.getIndex() + hit.getId())); + + if (lastHitSortValues != null) { + for (int i = 0; i < sort.length; i++) { + Comparable value = (Comparable) hit.getRawSortValues()[i]; + int cmp = value.compareTo(lastHitSortValues[i]) * reverseMuls[i]; + if (cmp != 0) { + assertThat(cmp, equalTo(1)); + break; + } + } + } + lastHitSortValues = hit.getRawSortValues(); + } + int len = response.getHits().getHits().length; + SearchHit last = response.getHits().getHits()[len - 1]; + if (lastSortValues != null) { + for (int i = 0; i < sort.length; i++) { + Comparable value = (Comparable) last.getSortValues()[i]; + int cmp = value.compareTo(lastSortValues[i]) * reverseMuls[i]; + if (cmp != 0) { + assertThat(cmp, equalTo(1)); + break; + } + } + } + assertThat(last.getSortValues().length, equalTo(sort.length)); + lastSortValues = last.getSortValues(); + searchRequest.source().searchAfter(last.getSortValues()); + response = client().search(searchRequest).get(); + } + assertThat(seen.size(), equalTo(expectedNumDocs)); + } + private String openPointInTime(String[] indices, TimeValue keepAlive) { OpenPointInTimeRequest request = new OpenPointInTimeRequest( indices, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index 55a56949b7dce..14d87dcb6a803 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -563,7 +563,7 @@ private TestIndexContext testIndex(MapperService mapperService, Client client) t directoryReader = DirectoryReader.open(directory); final LeafReaderContext leaf = directoryReader.leaves().get(0); - final QueryShardContext shardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, + final QueryShardContext shardContext = new QueryShardContext(shardId.id(), 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, new IndexSearcher(directoryReader), () -> nowInMillis, null, null, () -> true, null, emptyMap()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index 74042155ffd93..774c0d9694448 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -90,7 +90,7 @@ public void testDLS() throws Exception { Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); - QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); QueryShardContext queryShardContext = spy(realQueryShardContext); @@ -222,7 +222,7 @@ public void testDLSWithLimitedPermissions() throws Exception { Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); - QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); QueryShardContext queryShardContext = spy(realQueryShardContext); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 7cd1e85366989..25c73ad5b581a 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -241,6 +241,7 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t Map runtimeFields = emptyMap(); final QueryShardContext context = indexService.newQueryShardContext( shardId.id(), + 0, searcher, () -> { throw new UnsupportedOperationException(); }, null, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index ab0a57d6d3e10..90e57da2f39cb 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -88,7 +88,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase { @Before private void setup() { settings = createIndexSettings(); - queryShardContext = new QueryShardContext(0, settings, + queryShardContext = new QueryShardContext(0, 0, settings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, null, null, null, null, () -> 0L, null, null, () -> true, null, emptyMap()); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index d496756e19ee4..e4518dadf4e65 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -730,6 +730,7 @@ public void onIndexModule(IndexModule module) { module.setReaderWrapper(indexService -> new SecurityIndexReaderWrapper( shardId -> indexService.newQueryShardContext(shardId.id(), + 0, // we pass a null index reader, which is legal and will disable rewrite optimizations // based on index statistics, which is probably safer... null, diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index eb5b9b06bfb13..848fcc16eb8b2 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -897,7 +897,7 @@ protected final QueryShardContext createMockShardContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, + return new QueryShardContext(0, -1, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, null, xContentRegistry(), null, null, null, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()) { From e58ab36deb52dd7afb0c0723e244447ea8e1ac95 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 9 Dec 2020 11:29:19 +0100 Subject: [PATCH 02/10] unrelated change --- .../elasticsearch/common/io/stream/DelayableWriteable.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java index 1ced33ce6ed7f..1e58f12985c17 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java @@ -78,7 +78,7 @@ private DelayableWriteable() {} * {@code true} if the {@linkplain Writeable} is being stored in * serialized form, {@code false} otherwise. */ - public abstract boolean isSerialized(); + abstract boolean isSerialized(); private static class Referencing extends DelayableWriteable { private final T reference; @@ -109,7 +109,7 @@ public Serialized asSerialized(Reader reader, NamedWriteableRegistry regis } @Override - public boolean isSerialized() { + boolean isSerialized() { return false; } @@ -179,7 +179,7 @@ public Serialized asSerialized(Reader reader, NamedWriteableRegistry regis } @Override - public boolean isSerialized() { + boolean isSerialized() { return true; } From 3a456557e5409e9072cc940a37ea79499a831a08 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 9 Dec 2020 11:29:56 +0100 Subject: [PATCH 03/10] fix typo --- .../cluster/metadata/MetadataIndexAliasesService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 605c569a86dbf..5818034919d2d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -149,7 +149,7 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable System.currentTimeMillis(), null, emptyMap()), xContentRegistry); } }; From 53153aa287e6db1ef35528a36ea043e0fafc1a1e Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 9 Dec 2020 11:30:47 +0100 Subject: [PATCH 04/10] fix typo --- .../xpack/wildcard/mapper/WildcardFieldMapperTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 848fcc16eb8b2..b34b0689eddeb 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -897,7 +897,7 @@ protected final QueryShardContext createMockShardContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; - return new QueryShardContext(0, -1, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, null, xContentRegistry(), null, null, null, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()) { From fd55bdb62002f62514d486e89d07e8e4532a5e00 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 9 Dec 2020 11:36:13 +0100 Subject: [PATCH 05/10] unrelated change --- .../action/search/QueryPhaseResultConsumer.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java index 687adde514839..a84d5885a73c3 100644 --- a/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java +++ b/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java @@ -290,13 +290,9 @@ long ramBytesUsedQueryResult(QuerySearchResult result) { if (hasAggs == false) { return 0; } - if (result.aggregations().isSerialized()) { - return result.aggregations() - .asSerialized(InternalAggregations::readFrom, namedWriteableRegistry) - .ramBytesUsed(); - } else { - return result.aggregations().expand().getSerializedSize(); - } + return result.aggregations() + .asSerialized(InternalAggregations::readFrom, namedWriteableRegistry) + .ramBytesUsed(); } /** From 813ae3795f69c7adc212d3163bcaff1688d4d92b Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 9 Dec 2020 12:57:59 +0100 Subject: [PATCH 06/10] fix conflict --- .../search/aggregations/AggregatorTestCase.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index bca84b1c3cf67..af57af271ad72 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -272,7 +272,7 @@ protected AggregationContext createAggregationContext(IndexSearcher indexSearche .build(new IndexFieldDataCache.None(), breakerService); QueryShardContext queryShardContext = new QueryShardContext( 0, - 0, + -1, indexSettings, bigArrays, null, @@ -371,17 +371,6 @@ protected MapperService mapperServiceMock() { } /** - * Sub-tests that need a more complex index field data provider can override this - */ - protected TriFunction, IndexFieldData> getIndexFieldDataLookup( - MapperService mapperService, CircuitBreakerService circuitBreakerService) { - return (fieldType, s, searchLookup) -> fieldType.fielddataBuilder( - mapperService.getIndexSettings().getIndex().getName(), searchLookup) - .build(new IndexFieldDataCache.None(), circuitBreakerService); - } - - /** ->>>>>>> Stashed changes * Sub-tests that need scripting can override this method to provide a script service and pre-baked scripts */ protected ScriptService getMockScriptService() { From 5d8f0c92a688a28a011f615728184fed7c52c9b5 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 9 Dec 2020 18:29:41 +0100 Subject: [PATCH 07/10] fix typo --- .../src/main/java/org/elasticsearch/common/lucene/Lucene.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 865bb793b4438..6aa25cb95df91 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -585,7 +585,7 @@ private static SortField rewriteMergeSortField(SortField sortField) { ((SortedNumericSortField) sortField).getNumericType(), sortField.getReverse()); newSortField.setMissingValue(sortField.getMissingValue()); - return sortField; + return newSortField; } else if (sortField.getClass() == ShardDocSortField.class) { SortField newSortField = new SortField(sortField.getField(), SortField.Type.LONG, sortField.getReverse()); return newSortField; From 5af84f589bb04c8b63798f5d8c2ce1522af3b912 Mon Sep 17 00:00:00 2001 From: jimczi Date: Thu, 10 Dec 2020 21:24:02 +0100 Subject: [PATCH 08/10] fix rest test and remove leftover --- .../resources/rest-api-spec/test/search/90_search_after.yml | 2 +- .../java/org/elasticsearch/search/sort/FieldSortBuilder.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index 41c8ef3500a85..8111f3d50c294 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -240,5 +240,5 @@ index: test body: size: 1 - sort: [{ _shard_doc }] + sort: ["_shard_doc"] search_after: [ 0L ] diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index d7eb3d5aaf024..efc90370d2bf2 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -332,9 +332,6 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { if (DOC_FIELD_NAME.equals(fieldName)) { return reverse ? SORT_DOC_REVERSE : SORT_DOC; } else if (SHARD_DOC_FIELD_NAME.equals(fieldName)) { - if (context.getShardRequestIndex() == -1) { - throw new IllegalArgumentException(""); - } return new SortFieldAndFormat(new ShardDocSortField(context.getShardRequestIndex(), reverse), DocValueFormat.RAW); } From 9de14e92d93b5479c0852f61954db2e736ee6cc4 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 18 Dec 2020 09:00:07 +0100 Subject: [PATCH 09/10] fix compil --- .../fieldcaps/TransportFieldCapabilitiesIndexAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index e3d04dd95cc4f..8080c396b9d12 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -117,8 +117,8 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd final IndexShard indexShard = indexService.getShard(request.shardId().getId()); try (Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) { - final QueryShardContext queryShardContext = indexService.newQueryShardContext(shardId.id(), searcher, - request::nowInMillis, null, Collections.emptyMap()); + final QueryShardContext queryShardContext = indexService.newQueryShardContext(shardId.id(), 0, + searcher, request::nowInMillis, null, Collections.emptyMap()); if (canMatchShard(request, queryShardContext) == false) { return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false); From c7a279c3cd6c47161a75337e7b2f82778e0238c9 Mon Sep 17 00:00:00 2001 From: jimczi Date: Fri, 18 Dec 2020 09:47:12 +0100 Subject: [PATCH 10/10] fix mock in tests --- .../org/elasticsearch/search/DefaultSearchContextTests.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index b426df9e21895..04558e28312e9 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -106,9 +106,8 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), 0, anyObject(), anyObject(), anyString(), anyObject())).thenReturn( - queryShardContext - ); + when(indexService.newQueryShardContext(eq(shardId.id()), eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())) + .thenReturn(queryShardContext); MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); when(indexService.mapperService()).thenReturn(mapperService);