Test: add setting to change request timeout for rest client (#25201)

This commit adds a setting to change the request timeout for the rest client. This is useful as the
default timeout is 30s, which is also the same default for calls like cluster health. If both are
the same then the response from the cluster health api will not be received as the client usually
times out first making test failures harder to debug.

Relates #25185

Author: jaymode

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java

@@ -22,6 +22,8 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
@@ -46,5 +48,14 @@ public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate
     public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder().put(super.restClientSettings())
+            // increase the timeout so that we can actually see the result of failed cluster health
+            // calls that have a default timeout of 30s
+            .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "40s")
+            .build();
+    }
 }
 

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.client.RestClientBuilder;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -67,6 +68,7 @@
 public abstract class ESRestTestCase extends ESTestCase {
     public static final String TRUSTSTORE_PATH = "truststore.path";
     public static final String TRUSTSTORE_PASSWORD = "truststore.password";
+    public static final String CLIENT_RETRY_TIMEOUT = "client.retry.timeout";
 
     /**
      * Convert the entity from a {@link Response} into a map of maps.
@@ -338,6 +340,12 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE
             }
             builder.setDefaultHeaders(defaultHeaders);
         }
+
+        final String requestTimeoutString = settings.get(CLIENT_RETRY_TIMEOUT);
+        if (requestTimeoutString != null) {
+            final TimeValue maxRetryTimeout = TimeValue.parseTimeValue(requestTimeoutString, CLIENT_RETRY_TIMEOUT);
+            builder.setMaxRetryTimeoutMillis(Math.toIntExact(maxRetryTimeout.getMillis()));
+        }
         return builder.build();
     }