Backporting "Sense for VirtualBox and $HOME when deciding to turn on vagrant testing. (#24636)"

Author: jbaiera

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantSupportPlugin.groovy

@@ -0,0 +1,127 @@
+package org.elasticsearch.gradle.vagrant
+
+import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.process.ExecResult
+import org.gradle.process.internal.ExecException
+
+/**
+ * Global configuration for if Vagrant tasks are supported in this
+ * build environment.
+ */
+class VagrantSupportPlugin implements Plugin<Project> {
+
+    @Override
+    void apply(Project project) {
+        if (project.rootProject.ext.has('vagrantEnvChecksDone') == false) {
+            Map vagrantInstallation = getVagrantInstallation(project)
+            Map virtualBoxInstallation = getVirtualBoxInstallation(project)
+
+            project.rootProject.ext.vagrantInstallation = vagrantInstallation
+            project.rootProject.ext.virtualBoxInstallation = virtualBoxInstallation
+            project.rootProject.ext.vagrantSupported = vagrantInstallation.supported && virtualBoxInstallation.supported
+            project.rootProject.ext.vagrantEnvChecksDone = true
+
+            // Finding that HOME needs to be set when performing vagrant updates
+            String homeLocation = System.getenv("HOME")
+            if (project.rootProject.ext.vagrantSupported && homeLocation == null) {
+                throw new GradleException("Could not locate \$HOME environment variable. Vagrant is enabled " +
+                        "and requires \$HOME to be set to function properly.")
+            }
+        }
+
+        addVerifyInstallationTasks(project)
+    }
+
+    private Map getVagrantInstallation(Project project) {
+        try {
+            ByteArrayOutputStream pipe = new ByteArrayOutputStream()
+            ExecResult runResult = project.exec {
+                commandLine 'vagrant', '--version'
+                standardOutput pipe
+                ignoreExitValue true
+            }
+            String version = pipe.toString().trim()
+            if (runResult.exitValue == 0) {
+                if (version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/ || version ==~ /Vagrant 2\.[0-9]+\.[0-9]+/) {
+                    return [ 'supported' : true ]
+                } else {
+                    return [ 'supported' : false,
+                             'info' : "Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]" ]
+                }
+            } else {
+                return [ 'supported' : false,
+                         'info' : "Could not read installed vagrant version:\n" + version ]
+            }
+        } catch (ExecException e) {
+            // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
+            // Swallow error. Vagrant isn't installed. Don't halt the build here.
+            return [ 'supported' : false, 'info' : "Could not find vagrant: " + e.message ]
+        }
+    }
+
+    private Map getVirtualBoxInstallation(Project project) {
+        try {
+            ByteArrayOutputStream pipe = new ByteArrayOutputStream()
+            ExecResult runResult = project.exec {
+                commandLine 'vboxmanage', '--version'
+                standardOutput = pipe
+                ignoreExitValue true
+            }
+            String version = pipe.toString().trim()
+            if (runResult.exitValue == 0) {
+                try {
+                    String[] versions = version.split('\\.')
+                    int major = Integer.parseInt(versions[0])
+                    int minor = Integer.parseInt(versions[1])
+                    if ((major < 5) || (major == 5 && minor < 1)) {
+                        return [ 'supported' : false,
+                                 'info' : "Illegal version of virtualbox [${version}]. Need [5.1+]" ]
+                    } else {
+                        return [ 'supported' : true ]
+                    }
+                } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
+                    return [ 'supported' : false,
+                             'info' : "Unable to parse version of virtualbox [${version}]. Required [5.1+]" ]
+                }
+            } else {
+                return [ 'supported': false, 'info': "Could not read installed virtualbox version:\n" + version ]
+            }
+        } catch (ExecException e) {
+            // Exec still throws this if it cannot find the command, regardless if ignoreExitValue is set.
+            // Swallow error. VirtualBox isn't installed. Don't halt the build here.
+            return [ 'supported' : false, 'info' : "Could not find virtualbox: " + e.message ]
+        }
+    }
+
+    private void addVerifyInstallationTasks(Project project) {
+        createCheckVagrantVersionTask(project)
+        createCheckVirtualBoxVersionTask(project)
+    }
+
+    private void createCheckVagrantVersionTask(Project project) {
+        project.tasks.create('vagrantCheckVersion') {
+            description 'Check the Vagrant version'
+            group 'Verification'
+            doLast {
+                if (project.rootProject.vagrantInstallation.supported == false) {
+                    throw new InvalidUserDataException(project.rootProject.vagrantInstallation.info)
+                }
+            }
+        }
+    }
+
+    private void createCheckVirtualBoxVersionTask(Project project) {
+        project.tasks.create('virtualboxCheckVersion') {
+            description 'Check the Virtualbox version'
+            group 'Verification'
+            doLast {
+                if (project.rootProject.virtualBoxInstallation.supported == false) {
+                    throw new InvalidUserDataException(project.rootProject.virtualBoxInstallation.info)
+                }
+            }
+        }
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -228,43 +228,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils, copyBatsArchives, createVersionFile, createUpgradeFromFile
     }
 
-    private static void createCheckVagrantVersionTask(Project project) {
-        project.tasks.create('vagrantCheckVersion', Exec) {
-            description 'Check the Vagrant version'
-            group 'Verification'
-            commandLine 'vagrant', '--version'
-            standardOutput = new ByteArrayOutputStream()
-            doLast {
-                String version = standardOutput.toString().trim()
-                if ((version ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/) == false && (version ==~ /Vagrant 2\.[0-9]+\.[0-9]+/) == false) {
-                    throw new InvalidUserDataException("Illegal version of vagrant [${version}]. Need [Vagrant 1.8.6+]")
-                }
-            }
-        }
-    }
-
-    private static void createCheckVirtualBoxVersionTask(Project project) {
-        project.tasks.create('virtualboxCheckVersion', Exec) {
-            description 'Check the Virtualbox version'
-            group 'Verification'
-            commandLine 'vboxmanage', '--version'
-            standardOutput = new ByteArrayOutputStream()
-            doLast {
-                String version = standardOutput.toString().trim()
-                try {
-                    String[] versions = version.split('\\.')
-                    int major = Integer.parseInt(versions[0])
-                    int minor = Integer.parseInt(versions[1])
-                    if ((major < 5) || (major == 5 && minor < 1)) {
-                        throw new InvalidUserDataException("Illegal version of virtualbox [${version}]. Need [5.1+]")
-                    }
-                } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
-                    throw new InvalidUserDataException("Unable to parse version of virtualbox [${version}]. Required [5.1+]", e)
-                }
-            }
-        }
-    }
-
     private static void createPackagingTestTask(Project project) {
         project.tasks.create('packagingTest') {
             group 'Verification'
@@ -292,8 +255,6 @@ class VagrantTestPlugin implements Plugin<Project> {
         createCleanTask(project)
         createStopTask(project)
         createSmokeTestTask(project)
-        createCheckVagrantVersionTask(project)
-        createCheckVirtualBoxVersionTask(project)
         createPrepareVagrantTestEnvTask(project)
         createPackagingTestTask(project)
         createPlatformTestTask(project)

buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.vagrantsupport.properties

@@ -0,0 +1 @@
+implementation-class=org.elasticsearch.gradle.vagrant.VagrantSupportPlugin

plugins/repository-hdfs/build.gradle

@@ -29,6 +29,8 @@ esplugin {
   classname 'org.elasticsearch.repositories.hdfs.HdfsPlugin'
 }
 
+apply plugin: 'elasticsearch.vagrantsupport'
+
 versions << [
   'hadoop2': '2.8.1'
 ]
@@ -85,6 +87,7 @@ task krb5kdcUpdate(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) {
   subcommand 'update'
   boxName box
   environmentVars vagrantEnvVars
+  dependsOn "vagrantCheckVersion", "virtualboxCheckVersion"
 }
 
 task krb5kdcFixture(type: org.elasticsearch.gradle.test.VagrantFixture) {
@@ -135,7 +138,7 @@ task secureHdfsFixture(type: org.elasticsearch.gradle.test.AntFixture) {
   args miniHDFSArgs.toArray()
 }
 
-boolean fixtureSupported = false;
+boolean fixtureSupported = false
 if (Os.isFamily(Os.FAMILY_WINDOWS)) {
   // hdfs fixture will not start without hadoop native libraries on windows
   String nativePath = System.getenv("HADOOP_HOME")
@@ -163,48 +166,25 @@ if (fixtureSupported) {
   integTestRunner.systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic'
 }
 
-boolean secureFixtureSupported = false;
+boolean secureFixtureSupported = false
 if (fixtureSupported) {
-  // Only do secure fixture support if the regular fixture is supported,
-  // and if vagrant is installed. The ignoreExitValue on exec only matters
-  // in cases where the command can be found and successfully started. In
-  // situations where the vagrant command isn't able to be started at all
-  // (it's not installed) then Gradle still throws ExecException.
-  ByteArrayOutputStream pipe = new ByteArrayOutputStream()
-  try {
-    ExecResult runResult = exec {
-      commandLine 'vagrant', '--version'
-      standardOutput pipe
-      ignoreExitValue true
-    }
-    String output = pipe.toString().trim()
-    if (runResult.exitValue == 0) {
-      secureFixtureSupported = (output ==~ /Vagrant 1\.(8\.[6-9]|9\.[0-9])+/)
-    } else {
-      logger.warn("Could not read installed vagrant version:\n" + output)
-    }
-  } catch (org.gradle.process.internal.ExecException e) {
-    logger.warn("Could not find vagrant: " + e.message)
-    // Swallow error. Vagrant isn't installed. Leave secure fixture support off.
-  }
+  secureFixtureSupported = project.rootProject.vagrantSupported
 }
 
 // Create a Integration Test suite just for security based tests
-if (secureFixtureSupported && false) { // This fails due to a vagrant configuration issue - remove the false check to re-enable
-  // This must execute before the afterEvaluate block from integTestSecure
-  project.afterEvaluate {
-    Path elasticsearchKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("elasticsearch.keytab").toAbsolutePath()
-    Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
+// This must execute before the afterEvaluate block from integTestSecure
+project.afterEvaluate {
+  Path elasticsearchKT = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("keytabs").resolve("elasticsearch.keytab").toAbsolutePath()
+  Path krb5conf = project(':test:fixtures:krb5kdc-fixture').buildDir.toPath().resolve("conf").resolve("krb5.conf").toAbsolutePath()
 
     project.integTestSecureCluster.dependsOn(project.bundlePlugin)
     project.integTestSecure.clusterConfig.plugin(project.path)
     project.integTestSecure.clusterConfig.extraConfigFile("repository-hdfs/krb5.keytab", "${elasticsearchKT}")
-
     final String baseJvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
             " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
             " " + "-Djava.security.krb5.conf=${krb5conf}" +
             " " + System.getProperty('tests.jvm.argline', '')
-    final String jvmArgs
+  final String jvmArgs
     if (project.rootProject.ext.javaVersion == JavaVersion.VERSION_1_9) {
       jvmArgs = baseJvmArgs + " " + '--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED'
     } else {
@@ -214,18 +194,18 @@ if (secureFixtureSupported && false) { // This fails due to a vagrant configurat
     project.integTestSecure.clusterConfig.jvmArgs = jvmArgs
   }
 
-  RestIntegTestTask integTestSecure = project.tasks.create('integTestSecure', RestIntegTestTask.class) {
-    description = "Runs rest tests against an elasticsearch cluster with HDFS secured by MIT Kerberos."
-  }
+RestIntegTestTask integTestSecure = project.tasks.create('integTestSecure', RestIntegTestTask.class) {
+  description = "Runs rest tests against an elasticsearch cluster with HDFS secured by MIT Kerberos."
+}
 
-  integTestSecure.mustRunAfter(project.integTest)
+if (secureFixtureSupported) {
   project.check.dependsOn(integTestSecure)
 
   // Fixture dependencies
   integTestSecureCluster.dependsOn secureHdfsFixture, krb5kdcFixture
   integTestSecureRunner.systemProperty 'tests.rest.suite', 'secure_hdfs_repository'
 } else {
-  logger.warn("secured hdfsFixture is unsupported, please install Vagrant 1.8.6+ to enable")
+  integTestSecure.enabled = false
 }
 
 thirdPartyAudit.excludes = [

qa/vagrant/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.vagrantsupport'
 apply plugin: 'elasticsearch.vagrant'
 
 List<String> plugins = []