Return partial failures if search was cancelled. (#65467)

In #63520, we started cancelling searches that encounter shard failures and
don't allow partial results. In this case we return an 'all shards failed'
response, since there are no successful responses.

This PR proposes to return a 'partial shards failure' instead. The reasoning:
it's misleading to claim that 'all shards failed' when we only know at least
one shard failed (and preemptively cancelled the rest).

Author: jtibshirani

modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml

@@ -1,8 +1,5 @@
 ---
 "Response format for search failures":
-  - skip:
-      version: "all"
-      reason: "awaits fix of https://github.com/elastic/elasticsearch/pull/64382"
   - do:
       indices.create:
         index: source

server/src/internalClusterTest/java/org/elasticsearch/search/SearchCancellationIT.java

@@ -67,6 +67,7 @@
 import static org.elasticsearch.search.SearchCancellationIT.ScriptedBlockPlugin.SCRIPT_NAME;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
@@ -311,12 +312,12 @@ public void testCancelFailedSearchWhenPartialResultDisallowed() throws Exception
             .build());
         indexTestData();
         Thread searchThread = new Thread(() -> {
-            expectThrows(Exception.class, () -> {
+            SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () ->
                 client().prepareSearch("test")
                     .setSearchType(SearchType.QUERY_THEN_FETCH)
                     .setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap())))
-                    .setAllowPartialSearchResults(false).setSize(1000).get();
-            });
+                    .setAllowPartialSearchResults(false).setSize(1000).get());
+            assertThat(e.getMessage(), containsString("Partial shards failure"));
         });
         searchThread.start();
         try {

server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java

@@ -314,13 +314,16 @@ public boolean isForceExecution() {
         });
     }
 
+
     @Override
     public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPhase) {
-        /* This is the main search phase transition where we move to the next phase. At this point we check if there is
-         * at least one successful operation left and if so we move to the next phase. If not we immediately fail the
-         * search phase as "all shards failed"*/
-        if (successfulOps.get() == 0) { // we have 0 successful results that means we shortcut stuff and return a failure
-            final ShardOperationFailedException[] shardSearchFailures = ExceptionsHelper.groupBy(buildShardFailures());
+        /* This is the main search phase transition where we move to the next phase. If all shards
+         * failed or if there was a failure and partial results are not allowed, then we immediately
+         * fail. Otherwise we continue to the next phase.
+         */
+        ShardOperationFailedException[] shardSearchFailures = buildShardFailures();
+        if (shardSearchFailures.length == getNumShards()) {
+            shardSearchFailures = ExceptionsHelper.groupBy(shardSearchFailures);
             Throwable cause = shardSearchFailures.length == 0 ? null :
                 ElasticsearchException.guessRootCauses(shardSearchFailures[0].getCause())[0];
             logger.debug(() -> new ParameterizedMessage("All shards failed for phase: [{}]", getName()), cause);
@@ -331,7 +334,6 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha
             if (allowPartialResults == false && successfulOps.get() != getNumShards()) {
                 // check if there are actual failures in the atomic array since
                 // successful retries can reset the failures to null
-                ShardOperationFailedException[] shardSearchFailures = buildShardFailures();
                 if (shardSearchFailures.length > 0) {
                     if (logger.isDebugEnabled()) {
                         int numShardFailures = shardSearchFailures.length;