Merge branch 'master' into 2019-10-23-handle-negative-free-space

Author: DaveCTurner

.ci/os.ps1

@@ -5,10 +5,9 @@ If (-NOT ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdent
     exit
 }
 
-# CI configures these, uncoment if running manually 
-#
-# $env:ES_BUILD_JAVA="java12" 
-#$env:ES_RUNTIME_JAVA="java11" 
+$AppProps = ConvertFrom-StringData (Get-Content .ci/java-versions.properties -raw) 
+$env:ES_BUILD_JAVA=$AppProps.ES_BUILD_JAVA
+$env:ES_RUNTIME_JAVA=$AppProps.ES_RUNTIME_JAVA
 
 $ErrorActionPreference="Stop"
 $gradleInit = "C:\Users\$env:username\.gradle\init.d\"
@@ -27,10 +26,6 @@ New-Item -ItemType directory -Path \tmp
 
 $ErrorActionPreference="Continue"
 # TODO: remove the task exclusions once dependencies are set correctly and these don't run for Windows or buldiung the deb on windows is fixed
-& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --scan --console=plain destructiveDistroTest `
-   -x :distribution:packages:buildOssDeb `
-   -x :distribution:packages:buildDeb `
-   -x :distribution:packages:buildOssRpm `
-   -x :distribution:packages:buildRpm `
+& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --scan --console=plain destructiveDistroTest
 
 exit $LastExitCode

TESTING.asciidoc

@@ -43,6 +43,12 @@ supports a remote debugging option:
 ./gradlew run --debug-jvm
 ---------------------------------------------------------------------------
 
+This will instruct all JVMs (including any that run cli tools such as creating the keyring or adding users)
+to suspend and initiate a debug connection on port incrementing from `5005`.
+As such the IDE needs to be instructed to listen for connections on this port.
+Since we might run multiple JVMs as part of configuring and starting the cluster it's
+recommended to have the option to aut restart checked.
+
 ==== Distribution
 
 By default a node is started with the zip distribution.

buildSrc/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java

@@ -37,7 +37,7 @@
  * Since how to reap a given service is platform and service dependent, this tool
  * operates on system commands to execute. It takes a single argument, a directory
  * that will contain files with reaping commands. Each line in each file will be
- * executed with {@link Runtime#getRuntime()#exec}.
+ * executed with {@link Runtime#exec(String)}.
  *
  * The main method will wait indefinitely on the parent process (Gradle) by
  * reading from stdin. When Gradle shuts down, whether normally or abruptly, the

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -676,6 +676,10 @@ class BuildPlugin implements Plugin<Project> {
              */
             (javadoc.options as CoreJavadocOptions).addBooleanOption('html5', true)
         }
+        // ensure javadoc task is run with 'check'
+        project.pluginManager.withPlugin('lifecycle-base') {
+            project.tasks.getByName(LifecycleBasePlugin.CHECK_TASK_NAME).dependsOn(project.tasks.withType(Javadoc))
+        }
         configureJavadocJar(project)
     }
 
@@ -857,6 +861,9 @@ class BuildPlugin implements Plugin<Project> {
                         'tests.security.manager': 'true',
                         'jna.nosys': 'true'
 
+                //TODO remove once jvm.options are added to test system properties
+                test.systemProperty ('java.locale.providers','SPI,COMPAT')
+
                 // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation
                 if (System.getProperty('ignore.tests.seed') != null) {
                     nonInputProperties.systemProperty('tests.seed', project.property('testSeed'))
@@ -889,7 +896,6 @@ class BuildPlugin implements Plugin<Project> {
                 test.systemProperty('io.netty.noUnsafe', 'true')
                 test.systemProperty('io.netty.noKeySetOptimization', 'true')
                 test.systemProperty('io.netty.recycler.maxCapacityPerThread', '0')
-                test.systemProperty('io.netty.allocator.numDirectArenas', '0')
 
                 test.testLogging { TestLoggingContainer logging ->
                     logging.showExceptions = true

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -59,6 +59,7 @@
 import java.util.Map;
 import java.util.Random;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath;
 import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath;
@@ -187,6 +188,12 @@ private static List<Object> configureVM(Project project) {
         vagrant.setBox(box);
         vagrant.vmEnv("SYSTEM_JAVA_HOME", convertPath(project, vagrant, systemJdk, "", ""));
         vagrant.vmEnv("PATH", convertPath(project, vagrant, gradleJdk, "/bin:$PATH", "\\bin;$Env:PATH"));
+        // pass these along to get correct build scans
+        if (System.getenv("JENKINS_URL") != null) {
+            Stream.of("JOB_NAME", "JENKINS_URL", "BUILD_NUMBER", "BUILD_URL").forEach(name ->
+                vagrant.vmEnv(name, System.getenv(name))
+            );
+        }
         vagrant.setIsWindowsVM(isWindows(project));
 
         return Arrays.asList(systemJdk, gradleJdk);

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -24,6 +24,7 @@ import org.elasticsearch.gradle.testclusters.RestTestRunnerTask
 import org.elasticsearch.gradle.tool.Boilerplate
 import org.elasticsearch.gradle.tool.ClasspathUtils
 import org.gradle.api.DefaultTask
+import org.gradle.api.JavaVersion
 import org.gradle.api.Task
 import org.gradle.api.file.FileCopyDetails
 import org.gradle.api.tasks.Copy

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellTask.java

@@ -42,7 +42,7 @@ public JarHellTask() {
     @TaskAction
     public void runJarHellCheck() {
         LoggedExec.javaexec(getProject(), spec -> {
-            spec.classpath(getClasspath());
+            spec.environment("CLASSPATH", getClasspath().getAsPath());
             spec.setMain("org.elasticsearch.bootstrap.JarHell");
         });
     }

buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java

@@ -80,6 +80,7 @@ private List<String> getScript(boolean isWindows) {
         line.append(" --project-cache-dir ");
         line.append(isWindows ? convertWindowsPath(getProject(), cacheDir) : convertLinuxPath(getProject(), cacheDir));
         line.append(" -S");
+        line.append(" --parallel");
         line.append(" -D'org.gradle.logging.level'=" + getProject().getGradle().getStartParameter().getLogLevel());
         if (testClass != null) {
             line.append(" --tests=");

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -36,7 +36,7 @@ public Boolean getDebug() {
 
     @Override
     public void beforeStart() {
-        int debugPort = 8000;
+        int debugPort = 5005;
         int httpPort = 9200;
         int transportPort = 9300;
         Map<String, String> additionalSettings = System.getProperties().entrySet().stream()
@@ -57,7 +57,7 @@ public void beforeStart() {
                         "Running elasticsearch in debug mode, {} suspending until connected on debugPort {}",
                         node, debugPort
                     );
-                    node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=" + debugPort);
+                    node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=" + debugPort);
                     debugPort += 1;
                 }
             }

client/rest-high-level/build.gradle

@@ -129,8 +129,11 @@ testClusters.integTest {
 
   setting 'indices.lifecycle.poll_interval', '1000ms'
   keystore 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
+  extraConfigFile 'roles.yml', file('roles.yml')
   user username:  System.getProperty('tests.rest.cluster.username', 'test_user'),
-          password:  System.getProperty('tests.rest.cluster.password', 'test-password')
+          password:  System.getProperty('tests.rest.cluster.password', 'test-password'),
+          role:  System.getProperty('tests.rest.cluster.role', 'admin')
+  user username:  'admin_user', password: 'admin-password'
 
   extraConfigFile nodeCert.name, nodeCert
   extraConfigFile nodeTrustStore.name, nodeTrustStore