Remove deprecation warnings to prepare for Gradle 5 (sourceSets.main.output.classesDirs) (#30389)

* Remove deprecation warnings to prepare for Gradle 5

Gradle replaced `project.sourceSets.main.output.classesDir` of type
`File` with `project.sourceSets.main.output.classesDirs` of type
`FileCollection`
(see [SourceSetOutput](https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/java/org/gradle/api/tasks/SourceSetOutput.java))
Build output is now stored on a per language folder.

There are a few places where we use that, here's these and how it's
fixed:

- Randomized Test execution
    - look in all test folders ( pass the multi dir configuration to the
    ant runner )
    - DRY the task configuration by introducing `basedOn` for
      `RandomizedTestingTask` DSL
- Extend the naming convention test to support passing in multiple
  directories
- Fix the standalon test plugin, the dires were not passed trough,
  checked with a debuger and the statement had no affect due to a
  missing `=`.

Closes #30354

* Only check Java tests, PR feedback

- Name checker was ran for Groovy tests that don't adhere to the same
  convections causing the check to fail
- implement PR feedback

* Replace `add` with `addAll`

This worked because the list is passed to `project.files` that does the
right thing.

* Revert "Only check Java tests, PR feedback"

This reverts commit 9bd9389.

* Remove `basedOn` helper

* Bring some changes back

Previus revert accidentally reverted too much

* Fix negation

* add back public

* revert name check changes

* Revert "revert name check changes"

This reverts commit a2800c0.

* Pass all dirs to name check

Only run on Java for build-tools, this is safe because it's a self test.
It needs more work before we could pass in the Groovy classes as well as
these inherit from `GroovyTestCase`

* remove self tests from name check

The self complicates the task setup and disable real checks on
build-tools.
With this change there are no more self tests, and the build-tools tests
adhere to the conventions.
The self test will be replaced by gradle test kit, thus the addition of
the Gradle plugin builder plugin.

* First test to run a Gradle build

* Add tests that replace the name check self test

* Clean up integ test base class

* Always run tests

* Align with test naming conventions

* Make integ. test case inherit from unit test case

The check requires this

* Remove `import static org.junit.Assert.*`

Author: alpar-t

buildSrc/build.gradle

@@ -17,11 +17,13 @@
  * under the License.
  */
 
-import java.nio.file.Files
 
-import org.gradle.util.GradleVersion
+import java.nio.file.Files
 
-apply plugin: 'groovy'
+plugins {
+  id 'java-gradle-plugin'
+  id 'groovy'
+}
 
 group = 'org.elasticsearch.gradle'
 
@@ -83,9 +85,10 @@ repositories {
 }
 
 dependencies {
-  compile gradleApi()
   compile localGroovy()
   compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}"
+  compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+
   compile("junit:junit:${props.getProperty('junit')}") {
     transitive = false
   }
@@ -97,8 +100,10 @@ dependencies {
   compile 'de.thetaphi:forbiddenapis:2.5'
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
+  testCompile "junit:junit:${props.getProperty('junit')}"
 }
 
+
 // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
 // Use logging dependency instead
 // Gradle 4.3.1 stopped releasing the logging jars to jcenter, just use the last available one
@@ -113,14 +118,12 @@ dependencies {
  *****************************************************************************/
 // this will only happen when buildSrc is built on its own during build init
 if (project == rootProject) {
-
   repositories {
     if (System.getProperty("repos.mavenLocal") != null) {
       mavenLocal()
     }
     mavenCentral()
   }
-  test.exclude 'org/elasticsearch/test/NamingConventionsCheckBadClasses*'
 }
 
 /*****************************************************************************
@@ -145,9 +148,6 @@ if (project != rootProject) {
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
-  // test for elasticsearch.build tries to run with ES...
-  test.enabled = false
-
   // TODO: re-enable once randomizedtesting gradle code is published and removed from here
   licenseHeaders.enabled = false
 
@@ -158,14 +158,7 @@ if (project != rootProject) {
   }
 
   namingConventions {
-    testClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$UnitTestCase'
-    integTestClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$IntegTestCase'
-  }
-
-  task namingConventionsMain(type: org.elasticsearch.gradle.precommit.NamingConventionsTask) {
-    checkForTestsInMain = true
-    testClass = namingConventions.testClass
-    integTestClass = namingConventions.integTestClass
+    testClass = 'org.elasticsearch.gradle.test.GradleUnitTestCase'
+    integTestClass = 'org.elasticsearch.gradle.test.GradleIntegrationTestCase'
   }
-  precommit.dependsOn namingConventionsMain
 }

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy

@@ -89,7 +89,7 @@ class RandomizedTestingPlugin implements Plugin<Project> {
             description = 'Runs unit tests with the randomized testing framework'
             dependsOn oldTestTask.dependsOn, 'testClasses'
             classpath = oldTestTask.classpath
-            testClassesDir = oldTestTask.project.sourceSets.test.output.classesDir
+            testClassesDirs = oldTestTask.project.sourceSets.test.output.classesDirs
         }
 
         // hack so check task depends on custom test

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy

@@ -6,18 +6,20 @@ import groovy.xml.NamespaceBuilder
 import groovy.xml.NamespaceBuilderSupport
 import org.apache.tools.ant.BuildException
 import org.apache.tools.ant.DefaultLogger
+import org.apache.tools.ant.Project
 import org.apache.tools.ant.RuntimeConfigurable
 import org.apache.tools.ant.UnknownElement
+import org.elasticsearch.gradle.BuildPlugin
 import org.gradle.api.DefaultTask
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTreeElement
-import org.gradle.api.internal.tasks.options.Option
 import org.gradle.api.specs.Spec
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.options.Option
 import org.gradle.api.tasks.util.PatternFilterable
 import org.gradle.api.tasks.util.PatternSet
 import org.gradle.internal.logging.progress.ProgressLoggerFactory
@@ -43,8 +45,8 @@ class RandomizedTestingTask extends DefaultTask {
     @Input
     String parallelism = '1'
 
-    @InputDirectory
-    File testClassesDir
+    @Input
+    FileCollection testClassesDirs
 
     @Optional
     @Input
@@ -220,15 +222,15 @@ class RandomizedTestingTask extends DefaultTask {
                 listener = new DefaultLogger(
                         errorPrintStream: System.err,
                         outputPrintStream: System.out,
-                        messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO)
+                        messageOutputLevel: Project.MSG_INFO)
             } else {
                 // we want to buffer the info, and emit it if the test fails
                 antLoggingBuffer = new ByteArrayOutputStream()
                 PrintStream stream = new PrintStream(antLoggingBuffer, true, "UTF-8")
                 listener = new DefaultLogger(
                         errorPrintStream: stream,
                         outputPrintStream: stream,
-                        messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO)
+                        messageOutputLevel: Project.MSG_INFO)
             }
             project.ant.project.addBuildListener(listener)
         }
@@ -251,12 +253,10 @@ class RandomizedTestingTask extends DefaultTask {
                 if (argLine != null) {
                     jvmarg(line: argLine)
                 }
-                fileset(dir: testClassesDir) {
-                    for (String includePattern : patternSet.getIncludes()) {
-                        include(name: includePattern)
-                    }
-                    for (String excludePattern : patternSet.getExcludes()) {
-                        exclude(name: excludePattern)
+                testClassesDirs.each { testClassDir ->
+                    fileset(dir: testClassDir) {
+                        patternSet.getIncludes().each { include(name: it) }
+                        patternSet.getExcludes().each { exclude(name: it) }
                     }
                 }
                 for (Map.Entry<String, Object> prop : systemProperties) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -751,7 +751,7 @@ class BuildPlugin implements Plugin<Project> {
         project.extensions.add('additionalTest', { String name, Closure config ->
             RandomizedTestingTask additionalTest = project.tasks.create(name, RandomizedTestingTask.class)
             additionalTest.classpath = test.classpath
-            additionalTest.testClassesDir = test.testClassesDir
+            additionalTest.testClassesDirs = test.testClassesDirs
             additionalTest.configure(commonTestConfig(project))
             additionalTest.configure(config)
             test.dependsOn(additionalTest)

buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.groovy

@@ -0,0 +1,41 @@
+package org.elasticsearch.gradle;
+
+import groovy.lang.Closure;
+import org.gradle.api.GradleException;
+import org.gradle.api.Task;
+import org.gradle.api.tasks.Exec;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.stream.Collectors;
+
+/**
+ * A wrapper around gradle's Exec task to capture output and log on error.
+ */
+public class LoggedExec extends Exec {
+
+    protected ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+    public LoggedExec() {
+        if (getLogger().isInfoEnabled() == false) {
+            setStandardOutput(output);
+            setErrorOutput(output);
+            setIgnoreExitValue(true);
+            doLast(new Closure<Void>(this, this) {
+                public void doCall(Task it) throws IOException {
+                    if (getExecResult().getExitValue() != 0) {
+                        for (String line : output.toString("UTF-8").split("\\R")) {
+                            getLogger().error(line);
+                        }
+                        throw new GradleException(
+                                "Process \'" + getExecutable() + " " +
+                                        getArgs().stream().collect(Collectors.joining(" "))+
+                                        "\' finished with non-zero exit value " +
+                                        String.valueOf(getExecResult().getExitValue())
+                        );
+                    }
+                }
+            });
+        }
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.java

@@ -0,0 +1,50 @@
+package org.elasticsearch.gradle;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
+ */
+public class VersionProperties {
+    public static Version getElasticsearch() {
+        return elasticsearch;
+    }
+
+    public static String getLucene() {
+        return lucene;
+    }
+
+    public static Map<String, String> getVersions() {
+        return versions;
+    }
+
+    private static final Version elasticsearch;
+    private static final String lucene;
+    private static final Map<String, String> versions = new HashMap<String, String>();
+    static {
+        Properties props = getVersionProperties();
+        elasticsearch = Version.fromString(props.getProperty("elasticsearch"));
+        lucene = props.getProperty("lucene");
+        for (String property : props.stringPropertyNames()) {
+            versions.put(property, props.getProperty(property));
+        }
+    }
+
+    private static Properties getVersionProperties() {
+        Properties props = new Properties();
+        InputStream propsStream = VersionProperties.class.getResourceAsStream("/version.properties");
+        if (propsStream == null) {
+            throw new RuntimeException("/version.properties resource missing");
+        }
+        try {
+            props.load(propsStream);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return props;
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/VersionProperties.groovy

@@ -50,11 +50,11 @@ public class LoggerUsageTask extends LoggedExec {
                 List files = []
                 // But only if the source sets that will make them exist
                 if (project.sourceSets.findByName("main")) {
-                    files.add(project.sourceSets.main.output.classesDir)
+                    files.addAll(project.sourceSets.main.output.classesDirs.getFiles())
                     dependsOn project.tasks.classes
                 }
                 if (project.sourceSets.findByName("test")) {
-                    files.add(project.sourceSets.test.output.classesDir)
+                    files.addAll(project.sourceSets.test.output.classesDirs.getFiles())
                     dependsOn project.tasks.testClasses
                 }
                 /* In an extra twist, it isn't good enough that the source set