Merge branch 'master' into security_api_keys

Author: jaymode

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -51,6 +51,7 @@ import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.api.tasks.javadoc.Javadoc
 import org.gradle.internal.jvm.Jvm
 import org.gradle.process.ExecResult
+import org.gradle.process.ExecSpec
 import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
@@ -232,6 +233,95 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.java9Home = project.rootProject.ext.java9Home
     }
 
+    static void requireDocker(final Task task) {
+        final Project rootProject = task.project.rootProject
+        if (rootProject.hasProperty('requiresDocker') == false) {
+            /*
+             * This is our first time encountering a task that requires Docker. We will add an extension that will let us track the tasks
+             * that register as requiring Docker. We will add a delayed execution that when the task graph is ready if any such tasks are
+             * in the task graph, then we check two things:
+             *  - the Docker binary is available
+             *  - we can execute a Docker command that requires privileges
+             *
+             *  If either of these fail, we fail the build.
+             */
+            final boolean buildDocker
+            final String buildDockerProperty = System.getProperty("build.docker")
+            if (buildDockerProperty == null || buildDockerProperty == "true") {
+                buildDocker = true
+            } else if (buildDockerProperty == "false") {
+                buildDocker = false
+            } else {
+                throw new IllegalArgumentException(
+                        "expected build.docker to be unset or one of \"true\" or \"false\" but was [" + buildDockerProperty + "]")
+            }
+            rootProject.rootProject.ext.buildDocker = buildDocker
+            rootProject.rootProject.ext.requiresDocker = []
+            rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
+                // check if the Docker binary exists and record its path
+                final List<String> maybeDockerBinaries = ['/usr/bin/docker', '/usr/local/bin/docker']
+                final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
+
+                int exitCode
+                String dockerErrorOutput
+                if (dockerBinary == null) {
+                    exitCode = -1
+                    dockerErrorOutput = null
+                } else {
+                    // the Docker binary executes, check that we can execute a privileged command
+                    final ByteArrayOutputStream output = new ByteArrayOutputStream()
+                    final ExecResult result = LoggedExec.exec(rootProject, { ExecSpec it ->
+                        it.commandLine dockerBinary, "images"
+                        it.errorOutput = output
+                        it.ignoreExitValue = true
+                    })
+                    if (result.exitValue == 0) {
+                        return
+                    }
+                    exitCode = result.exitValue
+                    dockerErrorOutput = output.toString()
+                }
+                final List<String> tasks =
+                        ((List<Task>)rootProject.requiresDocker).findAll { taskGraph.hasTask(it) }.collect { "  ${it.path}".toString()}
+                if (tasks.isEmpty() == false) {
+                    /*
+                     * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
+                     * exist or because execution of a privileged Docker command failed.
+                     */
+                    String message
+                    if (dockerBinary == null) {
+                        message = String.format(
+                                Locale.ROOT,
+                                "Docker (checked [%s]) is required to run the following task%s: \n%s",
+                                maybeDockerBinaries.join(","),
+                                tasks.size() > 1 ? "s" : "",
+                                tasks.join('\n'))
+                    } else {
+                        assert exitCode > 0 && dockerErrorOutput != null
+                        message = String.format(
+                                Locale.ROOT,
+                                "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
+                                        "the problem is that Docker exited with exit code [%d] with standard error output [%s]",
+                                dockerBinary,
+                                tasks.size() > 1 ? "s" : "",
+                                tasks.join('\n'),
+                                exitCode,
+                                dockerErrorOutput.trim())
+                    }
+                    throw new GradleException(
+                            message + "\nyou can address this by attending to the reported issue, "
+                                    + "removing the offending tasks from being executed, "
+                                    + "or by passing -Dbuild.docker=false")
+                }
+            }
+        }
+        if (rootProject.buildDocker) {
+            rootProject.requiresDocker.add(task)
+        } else {
+            task.enabled = false
+        }
+    }
+
     private static String findCompilerJavaHome() {
         String compilerJavaHome = System.getenv('JAVA_HOME')
         final String compilerJavaProperty = System.getProperty('compiler.java')
@@ -773,7 +863,6 @@ class BuildPlugin implements Plugin<Project> {
         project.tasks.withType(RandomizedTestingTask) {task ->
             jvm "${project.runtimeJavaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel)
-            ifNoTests System.getProperty('tests.ifNoTests', 'fail')
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
 
@@ -785,10 +874,6 @@ class BuildPlugin implements Plugin<Project> {
                     task.shouldRunAfter testTask
                 }
             }
-            // no loose ends: check has to depend on all test tasks
-            project.tasks.matching {it.name == "check"}.all {
-                dependsOn(task)
-            }
 
             // TODO: why are we not passing maxmemory to junit4?
             jvmArg '-Xmx' + System.getProperty('tests.heap.size', '512m')

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -104,7 +104,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
      * format of the response is incompatible i.e. it is not a JSON object.
      */
     static shouldAddShardFailureCheck(String path) {
-        return path.startsWith('_cat') == false &&  path.startsWith('_xpack/ml/datafeeds/') == false
+        return path.startsWith('_cat') == false &&  path.startsWith('_ml/datafeeds/') == false
     }
 
     /**

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -129,6 +129,7 @@ public class PluginBuildPlugin extends BuildPlugin {
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
         integTest.mustRunAfter(project.precommit, project.test)
         project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
+        project.check.dependsOn(integTest)
     }
 
     /**

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -76,6 +76,22 @@ class ClusterConfiguration {
         }
     }
 
+    /**
+     * Whether the initial_master_nodes setting should be automatically derived from the nodes
+     * in the cluster. Only takes effect if all nodes in the cluster understand this setting
+     * and the discovery type is not explicitly set.
+     */
+    @Input
+    boolean autoSetInitialMasterNodes = true
+
+    /**
+     * Whether the file-based discovery provider should be automatically setup based on
+     * the nodes in the cluster. Only takes effect if no other hosts provider is already
+     * configured.
+     */
+    @Input
+    boolean autoSetHostsProvider = true
+
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
         " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.test
 
+import java.util.stream.Collectors
 import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
@@ -92,7 +93,8 @@ class ClusterFormationTasks {
             throw new Exception("tests.distribution=integ-test-zip is not supported")
         }
         configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
-        if (config.numBwcNodes > 0) {
+        boolean hasBwcNodes = config.numBwcNodes > 0
+        if (hasBwcNodes) {
             if (config.bwcVersion == null) {
                 throw new IllegalArgumentException("Must specify bwcVersion when numBwcNodes > 0")
             }
@@ -129,11 +131,23 @@ class ClusterFormationTasks {
             Object dependsOn
             if (node.nodeVersion.onOrAfter("6.5.0")) {
                 writeConfigSetup = { Map esConfig ->
-                    // Don't force discovery provider if one is set by the test cluster specs already
-                    if (esConfig.containsKey('discovery.zen.hosts_provider') == false) {
-                        esConfig['discovery.zen.hosts_provider'] = 'file'
+                    if (config.getAutoSetHostsProvider()) {
+                        // Don't force discovery provider if one is set by the test cluster specs already
+                        if (esConfig.containsKey('discovery.zen.hosts_provider') == false) {
+                            esConfig['discovery.zen.hosts_provider'] = 'file'
+                        }
+                        esConfig['discovery.zen.ping.unicast.hosts'] = []
+                    }
+                    boolean supportsInitialMasterNodes = hasBwcNodes == false || config.bwcVersion.onOrAfter("7.0.0")
+                    if (esConfig['discovery.type'] == null && config.getAutoSetInitialMasterNodes() && supportsInitialMasterNodes) {
+                        esConfig['cluster.initial_master_nodes'] = nodes.stream().map({ n ->
+                            if (n.config.settings['node.name'] == null) {
+                                return "node-" + n.nodeNum
+                            } else {
+                                return n.config.settings['node.name']
+                            }
+                        }).collect(Collectors.toList())
                     }
-                    esConfig['discovery.zen.ping.unicast.hosts'] = []
                     esConfig
                 }
                 dependsOn = startDependencies

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -19,14 +19,10 @@
 package org.elasticsearch.gradle.test
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
-import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.DefaultTask
-import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.execution.TaskExecutionAdapter
-import org.gradle.api.provider.Property
-import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskState
@@ -36,7 +32,6 @@ import org.gradle.plugins.ide.idea.IdeaPlugin
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.util.stream.Stream
-
 /**
  * A wrapper task around setting up a cluster and running rest tests.
  */

buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java

@@ -21,16 +21,18 @@ public class LoggedExec extends Exec {
 
     public LoggedExec() {
         ByteArrayOutputStream output = new ByteArrayOutputStream();
+        ByteArrayOutputStream error = new ByteArrayOutputStream();
         if (getLogger().isInfoEnabled() == false) {
             setStandardOutput(output);
-            setErrorOutput(output);
+            setErrorOutput(error);
             setIgnoreExitValue(true);
             doLast((unused) -> {
                     if (getExecResult().getExitValue() != 0) {
                         try {
-                            for (String line : output.toString("UTF-8").split("\\R")) {
-                                getLogger().error(line);
-                            }
+                            getLogger().error("Standard output:");
+                            getLogger().error(output.toString("UTF-8"));
+                            getLogger().error("Standard error:");
+                            getLogger().error(error.toString("UTF-8"));
                         } catch (UnsupportedEncodingException e) {
                             throw new GradleException("Failed to read exec output", e);
                         }
@@ -65,17 +67,19 @@ private static <T extends BaseExecSpec>  ExecResult genericExec(
             return function.apply(action);
         }
         ByteArrayOutputStream output = new ByteArrayOutputStream();
+        ByteArrayOutputStream error = new ByteArrayOutputStream();
         try {
             return function.apply(spec -> {
                 spec.setStandardOutput(output);
-                spec.setErrorOutput(output);
+                spec.setErrorOutput(error);
                 action.execute(spec);
             });
         } catch (Exception e) {
             try {
-                for (String line : output.toString("UTF-8").split("\\R")) {
-                    project.getLogger().error(line);
-                }
+                project.getLogger().error("Standard output:");
+                project.getLogger().error(output.toString("UTF-8"));
+                project.getLogger().error("Standard error:");
+                project.getLogger().error(error.toString("UTF-8"));
             } catch (UnsupportedEncodingException ue) {
                 throw new GradleException("Failed to read exec output", ue);
             }

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java

@@ -25,7 +25,6 @@
 import org.gradle.api.file.FileTree;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.OutputFile;
-import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.testing.Test;
 import org.gradle.api.tasks.util.PatternFilterable;
@@ -103,6 +102,22 @@ public void doCheck() throws IOException {
             final Map<String, Set<File>> classFilesPerRandomizedTestingTask = classFilesPerRandomizedTestingTask(allTestClassFiles);
             final Map<String, Set<File>> classFilesPerGradleTestTask = classFilesPerGradleTestTask();
 
+            Map<String, Set<Class<?>>> testClassesPerTask =
+                Stream.concat(
+                    classFilesPerGradleTestTask.entrySet().stream(),
+                    classFilesPerRandomizedTestingTask.entrySet().stream()
+                )
+                    .collect(
+                        Collectors.toMap(
+                            Map.Entry::getKey,
+                            entry -> entry.getValue().stream()
+                                .map(classes::get)
+                                .filter(implementsNamingConvention)
+                                .collect(Collectors.toSet())
+                        )
+                    );
+
+
             problems = collectProblems(
                 checkNoneExists(
                     "Test classes implemented by inner classes will not run",
@@ -118,6 +133,16 @@ public void doCheck() throws IOException {
                         .filter(this::seemsLikeATest)
                         .filter(implementsNamingConvention.negate())
                 ),
+                collectProblems(
+                    testClassesPerTask.entrySet().stream()
+                    .map( entry ->
+                        checkAtLeastOneExists(
+                            "test class in " + entry.getKey(),
+                            entry.getValue().stream()
+                        )
+                    )
+                    .collect(Collectors.joining())
+                ),
                 checkNoneExists(
                     "Test classes are not included in any enabled task (" +
                         Stream.concat(
@@ -215,7 +240,6 @@ private Class<? extends Task> getRandomizedTestingTask() {
     }
 
     @Input
-    @SkipWhenEmpty
     public Map<String, File> getTestClassNames() {
         if (testClassNames == null) {
             testClassNames = Boilerplate.getJavaSourceSets(getProject()).getByName("test").getOutput().getClassesDirs()
@@ -243,6 +267,14 @@ private String checkNoneExists(String message, Stream<? extends Class<?>> stream
         }
     }
 
+    private String checkAtLeastOneExists(String message, Stream<? extends Class<?>> stream) {
+        if (stream.findAny().isPresent()) {
+            return "";
+        } else {
+            return "Expected at least one " + message + ", but found none.\n";
+        }
+    }
+
     private boolean seemsLikeATest(Class<?> clazz) {
         try {
             ClassLoader classLoader = clazz.getClassLoader();