Skip to content

Commit 5416582

Browse files
jimczijimczi
authored
Filter original indices in shard level request (#78508)
Today the search action send the full list of original indices on every shard request. This change restricts the list to the concrete index of the shard or the alias that was used to resolve it. Relates #78314
1 parent 05b52de commit 5416582

File tree

62 files changed

+499
-255
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+499
-255
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
import org.apache.lucene.search.TopDocs;
1212
import org.apache.lucene.search.TotalHits;
1313
import org.apache.lucene.util.BytesRef;
14-
import org.elasticsearch.action.OriginalIndices;
1514
import org.elasticsearch.action.search.QueryPhaseResultConsumer;
1615
import org.elasticsearch.action.search.SearchPhaseController;
1716
import org.elasticsearch.action.search.SearchProgressListener;
@@ -183,9 +182,7 @@ public SearchPhaseController.ReducedQueryPhase reduceAggs(TermsList candidateLis
183182
new DocValueFormat[] { DocValueFormat.RAW }
184183
);
185184
result.aggregations(candidateList.get(i));
186-
result.setSearchShardTarget(
187-
new SearchShardTarget("node", new ShardId(new Index("index", "index"), i), null, OriginalIndices.NONE)
188-
);
185+
result.setSearchShardTarget(new SearchShardTarget("node", new ShardId(new Index("index", "index"), i), null));
189186
shards.add(result);
190187
}
191188
SearchRequest request = new SearchRequest();

‎client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountResponseTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ private static ShardSearchFailure createShardFailureTestItem() {
8080
String nodeId = randomAlphaOfLengthBetween(5, 10);
8181
String indexName = randomAlphaOfLengthBetween(5, 10);
8282
searchShardTarget = new SearchShardTarget(nodeId,
83-
new ShardId(new Index(indexName, IndexMetadata.INDEX_UUID_NA_VALUE), randomInt()), null, null);
83+
new ShardId(new Index(indexName, IndexMetadata.INDEX_UUID_NA_VALUE), randomInt()), null);
8484
}
8585
return new ShardSearchFailure(ex, searchShardTarget);
8686
}

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
package org.elasticsearch.index.rankeval;
1010

11-
import org.elasticsearch.action.OriginalIndices;
1211
import org.elasticsearch.common.Strings;
1312
import org.elasticsearch.common.bytes.BytesReference;
1413
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -65,7 +64,7 @@ public void testDCGAt() {
6564
rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i]));
6665
hits[i] = new SearchHit(i, Integer.toString(i), new Text(MapperService.SINGLE_MAPPING_NAME),
6766
Collections.emptyMap(), Collections.emptyMap());
68-
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE));
67+
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null));
6968
}
7069
DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
7170
assertEquals(EXPECTED_DCG, dcg.evaluate("id", hits, rated).metricScore(), DELTA);
@@ -116,7 +115,7 @@ public void testDCGAtSixMissingRatings() {
116115
}
117116
hits[i] = new SearchHit(i, Integer.toString(i), new Text(MapperService.SINGLE_MAPPING_NAME),
118117
Collections.emptyMap(), Collections.emptyMap());
119-
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE));
118+
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null));
120119
}
121120
DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
122121
EvalQueryQuality result = dcg.evaluate("id", hits, rated);
@@ -174,7 +173,7 @@ public void testDCGAtFourMoreRatings() {
174173
for (int i = 0; i < 4; i++) {
175174
hits[i] = new SearchHit(i, Integer.toString(i), new Text(MapperService.SINGLE_MAPPING_NAME),
176175
Collections.emptyMap(), Collections.emptyMap());
177-
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE));
176+
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null));
178177
}
179178
DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
180179
EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs);

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
package org.elasticsearch.index.rankeval;
1010

11-
import org.elasticsearch.action.OriginalIndices;
1211
import org.elasticsearch.common.bytes.BytesReference;
1312
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1413
import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -46,7 +45,7 @@ public static EvalQueryQuality randomEvalQueryQuality() {
4645
for (int i = 0; i < numberOfSearchHits; i++) {
4746
RatedSearchHit ratedSearchHit = RatedSearchHitTests.randomRatedSearchHit();
4847
// we need to associate each hit with an index name otherwise rendering will not work
49-
ratedSearchHit.getSearchHit().shard(new SearchShardTarget("_na_", new ShardId("index", "_na_", 0), null, OriginalIndices.NONE));
48+
ratedSearchHit.getSearchHit().shard(new SearchShardTarget("_na_", new ShardId("index", "_na_", 0), null));
5049
ratedHits.add(ratedSearchHit);
5150
}
5251
EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAlphaOfLength(10),

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
package org.elasticsearch.index.rankeval;
1010

11-
import org.elasticsearch.action.OriginalIndices;
1211
import org.elasticsearch.common.bytes.BytesReference;
1312
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1413
import org.elasticsearch.common.text.Text;
@@ -108,7 +107,7 @@ private SearchHit[] createSearchHits(List<RatedDocument> rated, Integer[] releva
108107
}
109108
hits[i] = new SearchHit(i, Integer.toString(i), new Text(MapperService.SINGLE_MAPPING_NAME),
110109
Collections.emptyMap(), Collections.emptyMap());
111-
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE));
110+
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null));
112111
}
113112
return hits;
114113
}

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
package org.elasticsearch.index.rankeval;
1010

11-
import org.elasticsearch.action.OriginalIndices;
1211
import org.elasticsearch.common.bytes.BytesReference;
1312
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1413
import org.elasticsearch.common.text.Text;
@@ -194,7 +193,7 @@ private static SearchHit[] createSearchHits(int from, int to, String index) {
194193
SearchHit[] hits = new SearchHit[to + 1 - from];
195194
for (int i = from; i <= to; i++) {
196195
hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap(), Collections.emptyMap());
197-
hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null, OriginalIndices.NONE));
196+
hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null));
198197
}
199198
return hits;
200199
}

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
package org.elasticsearch.index.rankeval;
1010

11-
import org.elasticsearch.action.OriginalIndices;
1211
import org.elasticsearch.common.bytes.BytesReference;
1312
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1413
import org.elasticsearch.common.text.Text;
@@ -104,7 +103,7 @@ public void testIgnoreUnlabeled() {
104103
// add an unlabeled search hit
105104
SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test"), 3);
106105
searchHits[2] = new SearchHit(2, "2", new Text(MapperService.SINGLE_MAPPING_NAME), Collections.emptyMap(), Collections.emptyMap());
107-
searchHits[2].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE));
106+
searchHits[2].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null));
108107

109108
EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", searchHits, rated);
110109
assertEquals((double) 2 / 3, evaluated.metricScore(), 0.00001);
@@ -124,7 +123,7 @@ public void testNoRatedDocs() throws Exception {
124123
for (int i = 0; i < 5; i++) {
125124
hits[i] = new SearchHit(i, i + "", new Text(MapperService.SINGLE_MAPPING_NAME),
126125
Collections.emptyMap(), Collections.emptyMap());
127-
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE));
126+
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null));
128127
}
129128
EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList());
130129
assertEquals(0.0d, evaluated.metricScore(), 0.00001);
@@ -246,7 +245,7 @@ private static SearchHit[] toSearchHits(List<RatedDocument> rated, String index)
246245
SearchHit[] hits = new SearchHit[rated.size()];
247246
for (int i = 0; i < rated.size(); i++) {
248247
hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap(), Collections.emptyMap());
249-
hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null, OriginalIndices.NONE));
248+
hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null));
250249
}
251250
return hits;
252251
}

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
package org.elasticsearch.index.rankeval;
1010

1111
import org.elasticsearch.ElasticsearchException;
12-
import org.elasticsearch.action.OriginalIndices;
1312
import org.elasticsearch.action.search.SearchPhaseExecutionException;
1413
import org.elasticsearch.action.search.ShardSearchFailure;
1514
import org.elasticsearch.cluster.block.ClusterBlockException;
@@ -60,7 +59,7 @@ public class RankEvalResponseTests extends ESTestCase {
6059
new IllegalArgumentException("Closed resource", new RuntimeException("Resource")),
6160
new SearchPhaseExecutionException("search", "all shards failed",
6261
new ShardSearchFailure[] { new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
63-
new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null, OriginalIndices.NONE)) }),
62+
new SearchShardTarget("node_1", new ShardId("foo", "_na_", 1), null)) }),
6463
new ElasticsearchException("Parsing failed",
6564
new ParsingException(9, 42, "Wrong state", new NullPointerException("Unexpected null value"))) };
6665

@@ -172,7 +171,7 @@ public void testToXContent() throws IOException {
172171
private static RatedSearchHit searchHit(String index, int docId, Integer rating) {
173172
SearchHit hit = new SearchHit(docId, docId + "", new Text(MapperService.SINGLE_MAPPING_NAME),
174173
Collections.emptyMap(), Collections.emptyMap());
175-
hit.shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null, OriginalIndices.NONE));
174+
hit.shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null));
176175
hit.score(1.0f);
177176
return new RatedSearchHit(hit, rating != null ? OptionalInt.of(rating) : OptionalInt.empty());
178177
}

‎modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RecallAtKTests.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
package org.elasticsearch.index.rankeval;
1010

11-
import org.elasticsearch.action.OriginalIndices;
1211
import org.elasticsearch.common.bytes.BytesReference;
1312
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1413
import org.elasticsearch.common.text.Text;
@@ -105,7 +104,7 @@ public void testNoRatedDocs() throws Exception {
105104
SearchHit[] hits = new SearchHit[k];
106105
for (int i = 0; i < k; i++) {
107106
hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap(), Collections.emptyMap());
108-
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null, OriginalIndices.NONE));
107+
hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0), null));
109108
}
110109

111110
EvalQueryQuality evaluated = (new RecallAtK()).evaluate("id", hits, Collections.emptyList());
@@ -227,7 +226,7 @@ private static SearchHit[] toSearchHits(List<RatedDocument> rated, String index)
227226
SearchHit[] hits = new SearchHit[rated.size()];
228227
for (int i = 0; i < rated.size(); i++) {
229228
hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap(), Collections.emptyMap());
230-
hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null, OriginalIndices.NONE));
229+
hits[i].shard(new SearchShardTarget("testnode", new ShardId(index, "uuid", 0), null));
231230
}
232231
return hits;
233232
}

‎server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -556,9 +556,10 @@ public void testSearchQueryThenFetch() throws Exception {
556556
assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
557557

558558
clearInterceptedActions();
559-
assertSameIndices(searchRequest, SearchTransportService.QUERY_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME);
559+
assertIndicesSubset(Arrays.asList(searchRequest.indices()), SearchTransportService.QUERY_ACTION_NAME,
560+
SearchTransportService.FETCH_ID_ACTION_NAME);
560561
//free context messages are not necessarily sent, but if they are, check their indices
561-
assertSameIndicesOptionalRequests(searchRequest, SearchTransportService.FREE_CONTEXT_ACTION_NAME);
562+
assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME);
562563
}
563564

564565
public void testSearchDfsQueryThenFetch() throws Exception {
@@ -577,10 +578,10 @@ public void testSearchDfsQueryThenFetch() throws Exception {
577578
assertThat(searchResponse.getHits().getTotalHits().value, greaterThan(0L));
578579

579580
clearInterceptedActions();
580-
assertSameIndices(searchRequest, SearchTransportService.DFS_ACTION_NAME, SearchTransportService.QUERY_ID_ACTION_NAME,
581-
SearchTransportService.FETCH_ID_ACTION_NAME);
581+
assertIndicesSubset(Arrays.asList(searchRequest.indices()), SearchTransportService.DFS_ACTION_NAME,
582+
SearchTransportService.QUERY_ID_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME);
582583
//free context messages are not necessarily sent, but if they are, check their indices
583-
assertSameIndicesOptionalRequests(searchRequest, SearchTransportService.FREE_CONTEXT_ACTION_NAME);
584+
assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME);
584585
}
585586

586587
private static void assertSameIndices(IndicesRequest originalRequest, String... actions) {
@@ -604,11 +605,22 @@ private static void assertSameIndices(IndicesRequest originalRequest, boolean op
604605
}
605606
}
606607
}
608+
607609
private static void assertIndicesSubset(List<String> indices, String... actions) {
610+
assertIndicesSubset(indices, false, actions);
611+
}
612+
613+
private static void assertIndicesSubsetOptionalRequests(List<String> indices, String... actions) {
614+
assertIndicesSubset(indices, true, actions);
615+
}
616+
617+
private static void assertIndicesSubset(List<String> indices, boolean optional, String... actions) {
608618
//indices returned by each bulk shard request need to be a subset of the original indices
609619
for (String action : actions) {
610620
List<TransportRequest> requests = consumeTransportRequests(action);
611-
assertThat("no internal requests intercepted for action [" + action + "]", requests.size(), greaterThan(0));
621+
if (optional == false) {
622+
assertThat("no internal requests intercepted for action [" + action + "]", requests.size(), greaterThan(0));
623+
}
612624
for (TransportRequest internalRequest : requests) {
613625
IndicesRequest indicesRequest = convertRequest(internalRequest);
614626
for (String index : indicesRequest.indices()) {

0 commit comments

Comments
 (0)