Skip to content

Commit 5e9c242

Browse files
dnhatndnhatn
authored
Disable sort optimization in search_after and scroll requests (#78230)
We can't enable the sort optimization with points in scroll requests until LUCENE-10119 is integrated.
1 parent a76fb0c commit 5e9c242

File tree

3 files changed

+154
-6
lines changed

3 files changed

+154
-6
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/search/searchafter/SearchAfterIT.java‎

Lines changed: 142 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,36 +9,50 @@
99
package org.elasticsearch.search.searchafter;
1010

1111
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
12+
import org.elasticsearch.action.bulk.BulkRequestBuilder;
1213
import org.elasticsearch.action.index.IndexRequest;
1314
import org.elasticsearch.action.index.IndexRequestBuilder;
15+
import org.elasticsearch.action.search.ClosePointInTimeAction;
16+
import org.elasticsearch.action.search.ClosePointInTimeRequest;
17+
import org.elasticsearch.action.search.OpenPointInTimeAction;
18+
import org.elasticsearch.action.search.OpenPointInTimeRequest;
1419
import org.elasticsearch.action.search.SearchPhaseExecutionException;
20+
import org.elasticsearch.action.search.SearchRequest;
1521
import org.elasticsearch.action.search.SearchRequestBuilder;
1622
import org.elasticsearch.action.search.SearchResponse;
1723
import org.elasticsearch.action.search.ShardSearchFailure;
1824
import org.elasticsearch.action.support.WriteRequest;
1925
import org.elasticsearch.cluster.metadata.IndexMetadata;
26+
import org.elasticsearch.common.Randomness;
2027
import org.elasticsearch.common.UUIDs;
2128
import org.elasticsearch.common.settings.Settings;
2229
import org.elasticsearch.common.xcontent.XContentBuilder;
30+
import org.elasticsearch.core.TimeValue;
31+
import org.elasticsearch.index.query.MatchAllQueryBuilder;
2332
import org.elasticsearch.rest.RestStatus;
2433
import org.elasticsearch.search.SearchHit;
34+
import org.elasticsearch.search.builder.PointInTimeBuilder;
35+
import org.elasticsearch.search.builder.SearchSourceBuilder;
36+
import org.elasticsearch.search.sort.FieldSortBuilder;
2537
import org.elasticsearch.search.sort.SortBuilders;
2638
import org.elasticsearch.search.sort.SortOrder;
2739
import org.elasticsearch.test.ESIntegTestCase;
2840
import org.hamcrest.Matchers;
2941

30-
import java.util.List;
3142
import java.util.ArrayList;
32-
import java.util.Comparator;
33-
import java.util.Collections;
3443
import java.util.Arrays;
44+
import java.util.Collections;
45+
import java.util.Comparator;
46+
import java.util.List;
3547

36-
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
48+
import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
3749
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
3850
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
51+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
3952
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
4053
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
4154
import static org.hamcrest.Matchers.arrayContaining;
55+
import static org.hamcrest.Matchers.arrayWithSize;
4256
import static org.hamcrest.Matchers.containsString;
4357
import static org.hamcrest.Matchers.equalTo;
4458

@@ -395,4 +409,128 @@ private List<Object> convertSortValues(List<Object> sortValues) {
395409
}
396410
return converted;
397411
}
412+
413+
public void testScrollAndSearchAfterWithBigIndex() {
414+
int numDocs = randomIntBetween(5000, 10000);
415+
List<Long> timestamps = new ArrayList<>();
416+
long currentTime = randomLongBetween(0, 1000);
417+
for (int i = 0; i < numDocs; i++) {
418+
int copies = randomIntBetween(0, 100) <= 5 ? randomIntBetween(2, 5) : 1;
419+
for (int j = 0; j < copies; j++) {
420+
timestamps.add(currentTime);
421+
}
422+
currentTime += randomIntBetween(1, 10);
423+
}
424+
final Settings.Builder indexSettings = Settings.builder()
425+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5));
426+
if (randomBoolean()) {
427+
indexSettings.put("sort.field", "timestamp").put("sort.order", randomFrom("desc", "asc"));
428+
}
429+
assertAcked(client().admin().indices().prepareCreate("test")
430+
.setSettings(indexSettings)
431+
.setMapping("{\"properties\":{\"timestamp\":{\"type\": \"date\", \"format\": \"epoch_millis\"}}}"));
432+
Randomness.shuffle(timestamps);
433+
final BulkRequestBuilder bulk = client().prepareBulk();
434+
bulk.setRefreshPolicy(IMMEDIATE);
435+
for (long timestamp : timestamps) {
436+
bulk.add(new IndexRequest("test").source("timestamp", timestamp));
437+
}
438+
bulk.get();
439+
Collections.sort(timestamps);
440+
// scroll with big index
441+
{
442+
SearchResponse resp = client().prepareSearch("test")
443+
.setSize(randomIntBetween(50, 100))
444+
.setQuery(new MatchAllQueryBuilder())
445+
.addSort(new FieldSortBuilder("timestamp"))
446+
.setScroll(TimeValue.timeValueMinutes(5))
447+
.get();
448+
try {
449+
int foundHits = 0;
450+
do {
451+
for (SearchHit hit : resp.getHits().getHits()) {
452+
assertNotNull(hit.getSourceAsMap());
453+
final Object timestamp = hit.getSourceAsMap().get("timestamp");
454+
assertNotNull(timestamp);
455+
assertThat(((Number) timestamp).longValue(), equalTo(timestamps.get(foundHits)));
456+
foundHits++;
457+
}
458+
resp = client().prepareSearchScroll(resp.getScrollId())
459+
.setScroll(TimeValue.timeValueMinutes(5))
460+
.get();
461+
} while (resp.getHits().getHits().length > 0);
462+
assertThat(foundHits, equalTo(timestamps.size()));
463+
} finally {
464+
client().prepareClearScroll().addScrollId(resp.getScrollId()).get();
465+
}
466+
}
467+
// search_after with sort with point in time
468+
String pitID;
469+
{
470+
OpenPointInTimeRequest openPITRequest = new OpenPointInTimeRequest("test").keepAlive(TimeValue.timeValueMinutes(5));
471+
pitID = client().execute(OpenPointInTimeAction.INSTANCE, openPITRequest).actionGet().getPointInTimeId();
472+
SearchRequest searchRequest = new SearchRequest("test")
473+
.source(new SearchSourceBuilder()
474+
.pointInTimeBuilder(new PointInTimeBuilder(pitID).setKeepAlive(TimeValue.timeValueMinutes(5)))
475+
.sort("timestamp"));
476+
searchRequest.source().size(randomIntBetween(50, 100));
477+
SearchResponse resp = client().search(searchRequest).actionGet();
478+
try {
479+
int foundHits = 0;
480+
do {
481+
Object[] after = null;
482+
for (SearchHit hit : resp.getHits().getHits()) {
483+
assertNotNull(hit.getSourceAsMap());
484+
final Object timestamp = hit.getSourceAsMap().get("timestamp");
485+
assertNotNull(timestamp);
486+
assertThat(((Number) timestamp).longValue(), equalTo(timestamps.get(foundHits)));
487+
after = hit.getSortValues();
488+
foundHits++;
489+
}
490+
searchRequest.source().size(randomIntBetween(50, 100));
491+
assertNotNull(after);
492+
assertThat("Sorted by timestamp and pit tier breaker", after, arrayWithSize(2));
493+
searchRequest.source().searchAfter(after);
494+
resp = client().search(searchRequest).actionGet();
495+
} while (resp.getHits().getHits().length > 0);
496+
assertThat(foundHits, equalTo(timestamps.size()));
497+
} finally {
498+
client().execute(ClosePointInTimeAction.INSTANCE, new ClosePointInTimeRequest(pitID)).actionGet();
499+
}
500+
}
501+
502+
// search_after without sort with point in time
503+
{
504+
OpenPointInTimeRequest openPITRequest = new OpenPointInTimeRequest("test").keepAlive(TimeValue.timeValueMinutes(5));
505+
pitID = client().execute(OpenPointInTimeAction.INSTANCE, openPITRequest).actionGet().getPointInTimeId();
506+
SearchRequest searchRequest = new SearchRequest("test")
507+
.source(new SearchSourceBuilder()
508+
.pointInTimeBuilder(new PointInTimeBuilder(pitID).setKeepAlive(TimeValue.timeValueMinutes(5)))
509+
.sort(SortBuilders.pitTiebreaker()));
510+
searchRequest.source().size(randomIntBetween(50, 100));
511+
SearchResponse resp = client().search(searchRequest).actionGet();
512+
List<Long> foundSeqNos = new ArrayList<>();
513+
try {
514+
do {
515+
Object[] after = null;
516+
for (SearchHit hit : resp.getHits().getHits()) {
517+
assertNotNull(hit.getSourceAsMap());
518+
final Object timestamp = hit.getSourceAsMap().get("timestamp");
519+
assertNotNull(timestamp);
520+
foundSeqNos.add(((Number) timestamp).longValue());
521+
after = hit.getSortValues();
522+
}
523+
searchRequest.source().size(randomIntBetween(50, 100));
524+
assertNotNull(after);
525+
assertThat("sorted by pit tie breaker", after, arrayWithSize(1));
526+
searchRequest.source().searchAfter(after);
527+
resp = client().search(searchRequest).actionGet();
528+
} while (resp.getHits().getHits().length > 0);
529+
Collections.sort(foundSeqNos);
530+
assertThat(foundSeqNos, equalTo(timestamps));
531+
} finally {
532+
client().execute(ClosePointInTimeAction.INSTANCE, new ClosePointInTimeRequest(pitID)).actionGet();
533+
}
534+
}
535+
}
398536
}

‎server/src/main/java/org/elasticsearch/search/query/QueryPhase.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.apache.lucene.search.Query;
2020
import org.apache.lucene.search.ScoreDoc;
2121
import org.apache.lucene.search.Sort;
22+
import org.apache.lucene.search.SortField;
2223
import org.apache.lucene.search.TopDocs;
2324
import org.apache.lucene.search.TotalHits;
2425
import org.elasticsearch.action.search.SearchShardTask;
@@ -136,6 +137,14 @@ static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExe
136137
assert query == searcher.rewrite(query); // already rewritten
137138

138139
final ScrollContext scrollContext = searchContext.scrollContext();
140+
if (searchContext.sort() != null && searchContext.sort().sort.getSort().length == 1) {
141+
// TODO: Enable the sort optimization with point for search_after and scroll requests when LUCENE-10119 is integrated.
142+
if ((scrollContext != null && scrollContext.lastEmittedDoc != null) || searchContext.searchAfter() != null) {
143+
for (SortField sortField : searchContext.sort().sort.getSort()) {
144+
sortField.setOptimizeSortWithPoints(false);
145+
}
146+
}
147+
}
139148
if (scrollContext != null) {
140149
if (scrollContext.totalHits == null) {
141150
// first round

‎server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -729,11 +729,12 @@ public void testNumericSortOptimization() throws Exception {
729729
searchContext.trackTotalHitsUpTo(10);
730730
searchContext.setSize(10);
731731
QueryPhase.executeInternal(searchContext);
732-
assertTrue(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints());
732+
assertFalse(searchContext.sort().sort.getSort()[0].getOptimizeSortWithPoints());
733733
final TopDocs topDocs = searchContext.queryResult().topDocs().topDocs;
734734
long firstResult = (long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0];
735735
assertThat(firstResult, greaterThan(afterValue));
736-
assertSortResults(topDocs, numDocs, false);
736+
searchContext.sort().sort.getSort()[0].setOptimizeSortWithPoints(true);
737+
// assertSortResults(topDocs, numDocs, false);
737738
}
738739

739740
// 3. Test sort optimization on long field + date field

0 commit comments

Comments
 (0)