Merge branch 'master' into security_api_keys

Author: Yogesh Gaikwad

TESTING.asciidoc

@@ -150,8 +150,9 @@ Default value provided below in [brackets].
 
 === Load balancing and caches.
 
-By default the tests run on up to 4 JVMs based on the number of cores. If you
-want to explicitly specify the number of JVMs you can do so on the command
+By default the tests run on multiple processes using all the available cores on all 
+available CPUs. Not including hyper-threading.
+If you want to explicitly specify the number of JVMs you can do so on the command
 line:
 
 ----------------------------

build.gradle

@@ -25,6 +25,9 @@ import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
+import com.carrotsearch.gradle.junit4.RandomizedTestingTask
+
+import java.util.function.Predicate
 
 plugins {
     id 'com.gradle.build-scan' version '1.13.2'
@@ -622,3 +625,19 @@ allprojects {
     }
   }
 }
+
+allprojects {
+  task checkPart1
+  task checkPart2 
+  tasks.matching { it.name == "check" }.all { check ->
+    if (check.path.startsWith(":x-pack:")) {
+      checkPart2.dependsOn check
+    } else {
+      checkPart1.dependsOn check
+    }
+  } 
+}
+
+
+
+

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -773,9 +773,32 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     static void applyCommonTestConfig(Project project) {
+        String defaultParallel = 'auto'
+        // Count physical cores on any Linux distro ( don't count hyper-threading )
+        if (project.file("/proc/cpuinfo").exists()) {
+            Map<String, Integer> socketToCore = [:]
+            String currentID = ""
+            project.file("/proc/cpuinfo").readLines().forEach({ line ->
+                if (line.contains(":")) {
+                    List<String> parts = line.split(":", 2).collect({it.trim()})
+                    String name = parts[0], value = parts[1]
+                    // the ID of the CPU socket
+                    if (name == "physical id") {
+                        currentID = value
+                    }
+                    // Number  of cores not including hyper-threading
+                    if (name == "cpu cores") {
+                        assert currentID.isEmpty() == false
+                        socketToCore[currentID] = Integer.valueOf(value)
+                        currentID = ""
+                    }
+                }
+            })
+            defaultParallel = socketToCore.values().sum().toString();
+        }
         project.tasks.withType(RandomizedTestingTask) {
             jvm "${project.runtimeJavaHome}/bin/java"
-            parallelism System.getProperty('tests.jvms', 'auto')
+            parallelism System.getProperty('tests.jvms', defaultParallel)
             ifNoTests System.getProperty('tests.ifNoTests', 'fail')
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -24,7 +24,9 @@ import org.apache.rat.license.SimpleLicenseFamily
 import org.elasticsearch.gradle.AntTask
 import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.SkipWhenEmpty
 import org.gradle.api.tasks.SourceSet
 
 import java.nio.file.Files
@@ -39,12 +41,6 @@ public class LicenseHeadersTask extends AntTask {
     @OutputFile
     File reportFile = new File(project.buildDir, 'reports/licenseHeaders/rat.log')
 
-    /**
-     * The list of java files to check. protected so the afterEvaluate closure in the
-     * constructor can write to it.
-     */
-    protected List<FileCollection> javaFiles
-
     /** Allowed license families for this project. */
     @Input
     List<String> approvedLicenses = ['Apache', 'Generated']
@@ -65,11 +61,16 @@ public class LicenseHeadersTask extends AntTask {
 
     LicenseHeadersTask() {
         description = "Checks sources for missing, incorrect, or unacceptable license headers"
-        // Delay resolving the dependencies until after evaluation so we pick up generated sources
-        project.afterEvaluate {
-            javaFiles = project.sourceSets.collect({it.allJava})
-            inputs.files(javaFiles)
-        }
+    }
+
+    /**
+     * The list of java files to check. protected so the afterEvaluate closure in the
+     * constructor can write to it.
+     */
+    @InputFiles
+    @SkipWhenEmpty
+    public List<FileCollection> getJavaFiles() {
+        return project.sourceSets.collect({it.allJava})
     }
 
     /**
@@ -97,9 +98,8 @@ public class LicenseHeadersTask extends AntTask {
         Files.deleteIfExists(reportFile.toPath())
 
         // run rat, going to the file
-        List<FileCollection> input = javaFiles
         ant.ratReport(reportFile: reportFile.absolutePath, addDefaultLicenseMatchers: true) {
-            for (FileCollection dirSet : input) {
+            for (FileCollection dirSet : javaFiles) {
                for (File dir: dirSet.srcDirs) {
                    // sometimes these dirs don't exist, e.g. site-plugin has no actual java src/main...
                    if (dir.exists()) {