Do not search locally if remote index pattern resolves to no indices (#25436)

This commit changes how we determine if there were any remote indices that a search should have
been executed against. Previously, we used the list of remote shard iterators but if the remote
index pattern resolved to no indices there would be no remote shard iterators even though the
request specified remote indices. The map of remote cluster names to the original indices is used
instead so that we can determine if there were remote indices even when there are no remote shard
iterators.

Closes #25426

Author: jaymode

core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -33,7 +33,6 @@
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -184,7 +183,7 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<
             searchRequest.indices(), idx -> indexNameExpressionResolver.hasIndexOrAlias(idx, clusterState));
         OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
         if (remoteClusterIndices.isEmpty()) {
-            executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, Collections.emptyList(),
+            executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, remoteClusterIndices, Collections.emptyList(),
                 (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener);
         } else {
             remoteClusterService.collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(),
@@ -193,7 +192,7 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<
                     Map<String, AliasFilter> remoteAliasFilters = new HashMap<>();
                     BiFunction<String, String, DiscoveryNode> clusterNodeLookup = processRemoteShards(searchShardsResponses,
                         remoteClusterIndices, remoteShardIterators, remoteAliasFilters);
-                    executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, remoteShardIterators,
+                    executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices, remoteClusterIndices, remoteShardIterators,
                         clusterNodeLookup, clusterState, remoteAliasFilters, listener);
                 }, listener::onFailure));
         }
@@ -249,16 +248,16 @@ static BiFunction<String, String, DiscoveryNode> processRemoteShards(Map<String,
     }
 
     private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, SearchRequest searchRequest, OriginalIndices localIndices,
-                               List<SearchShardIterator> remoteShardIterators, BiFunction<String, String, DiscoveryNode> remoteConnections,
-                               ClusterState clusterState, Map<String, AliasFilter> remoteAliasMap,
-                               ActionListener<SearchResponse> listener) {
+                               Map<String, OriginalIndices> remoteClusterIndices, List<SearchShardIterator> remoteShardIterators,
+                               BiFunction<String, String, DiscoveryNode> remoteConnections, ClusterState clusterState,
+                               Map<String, AliasFilter> remoteAliasMap, ActionListener<SearchResponse> listener) {
 
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
         // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
         // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
         // of just for the _search api
         final Index[] indices;
-        if (localIndices.indices().length == 0 && remoteShardIterators.size() > 0) {
+        if (localIndices.indices().length == 0 && remoteClusterIndices.isEmpty() == false) {
             indices = Index.EMPTY_ARRAY; // don't search on _all if only remote indices were specified
         } else {
             indices = indexNameExpressionResolver.concreteIndices(clusterState, searchRequest.indicesOptions(),

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/50_missing.yml

@@ -0,0 +1,44 @@
+---
+"Search with missing remote index pattern":
+  - do:
+      catch: "request"
+      search:
+        index: "my_remote_cluster:foo"
+
+  - do:
+      search:
+        index: "my_remote_cluster:fooo*"
+  - match: { _shards.total: 0 }
+  - match: { hits.total: 0 }
+
+  - do:
+      search:
+        index: "*:foo*"
+
+  - match: { _shards.total: 0 }
+  - match: { hits.total: 0 }
+
+  - do:
+      search:
+        index: "my_remote_cluster:test_index,my_remote_cluster:foo*"
+        body:
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+
+  - do:
+      catch: "request"
+      search:
+        index: "my_remote_cluster:test_index,my_remote_cluster:foo"
+        body:
+          aggs:
+            cluster:
+              terms:
+                field: f1.keyword