Build: allow the amount of time we wait for a node to be configured (#28403)

This commit allows for configuration of the amount of time we wait for a node to startup. This is needed as some QA tests with plugins and tribe timeout when starting the tribe
node. Even regular node startup time with plugins is starting to approach the limit of 30 seconds.

Additionally, this commit provides better feedback when a wait has failed. The code checks to
ensure all expected files exist for each node; if they do not then we consider the wait task as
having failed. Prior to this change, when there was one or more missing file the build would
continue and attempt to execute the wait condition that typically makes an HTTP request to the
cluster. The output of this type of failure does include which files exist and which do not but
this change makes it clearer that the actual HTTP call did not time out, but the failure was before
the call was even made.

Author: jaymode

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -123,6 +123,14 @@ class ClusterConfiguration {
         return tmpFile.exists()
     }
 
+    /**
+     * The maximum number of seconds to wait for nodes to complete startup, which includes writing
+     * the ports files for the transports and the pid file. This wait time occurs before the wait
+     * condition is executed.
+     */
+    @Input
+    int nodeStartupWaitSeconds = 30
+
     public ClusterConfiguration(Project project) {
         this.project = project
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -24,7 +24,6 @@ import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.MetaPluginBuildPlugin
-import org.elasticsearch.gradle.plugin.MetaPluginPropertiesExtension
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
@@ -120,7 +119,7 @@ class ClusterFormationTasks {
             startTasks.add(configureNode(project, prefix, runner, dependsOn, node, config, distro, nodes.get(0)))
         }
 
-        Task wait = configureWaitTask("${prefix}#wait", project, nodes, startTasks)
+        Task wait = configureWaitTask("${prefix}#wait", project, nodes, startTasks, config.nodeStartupWaitSeconds)
         runner.dependsOn(wait)
 
         return nodes
@@ -581,10 +580,10 @@ class ClusterFormationTasks {
         return start
     }
 
-    static Task configureWaitTask(String name, Project project, List<NodeInfo> nodes, List<Task> startTasks) {
+    static Task configureWaitTask(String name, Project project, List<NodeInfo> nodes, List<Task> startTasks, int waitSeconds) {
         Task wait = project.tasks.create(name: name, dependsOn: startTasks)
         wait.doLast {
-            ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${name}") {
+            ant.waitfor(maxwait: "${waitSeconds}", maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${name}") {
                 or {
                     for (NodeInfo node : nodes) {
                         resourceexists {
@@ -614,6 +613,17 @@ class ClusterFormationTasks {
                 waitFailed(project, nodes, logger, 'Failed to start elasticsearch')
             }
 
+            // make sure all files exist otherwise we haven't fully started up
+            boolean missingFile = false
+            for (NodeInfo node : nodes) {
+                missingFile |= node.pidFile.exists() == false
+                missingFile |= node.httpPortsFile.exists() == false
+                missingFile |= node.transportPortsFile.exists() == false
+            }
+            if (missingFile) {
+                waitFailed(project, nodes, logger, 'Elasticsearch did not complete startup in time allotted')
+            }
+
             // go through each node checking the wait condition
             for (NodeInfo node : nodes) {
                 // first bind node info to the closure, then pass to the ant runner so we can get good logging