Use TestFixturesPlugin to Run Minio in Tests (#37852) (#38973)

* Use TestFixturesPlugin to Run Minio in Tests  (#37852)

* Use TestFixturesPlugin to Run Minio in Tests

* Closes #37680
* Closes #37783

* Fix S3 Repository ITs When Docker is not Available (#37878)

* Disable Minio fixture and tests that require it when fixtures are disabled or Docker is not available
* Relates #37852

* add explicit composeUp dependsOn

Author: talevy

plugins/repository-s3/build.gradle

@@ -1,13 +1,10 @@
-import org.apache.tools.ant.taskdefs.condition.Os
-import org.elasticsearch.gradle.LoggedExec
+import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.AntFixture
 import org.elasticsearch.gradle.test.ClusterConfiguration
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 
-import java.lang.reflect.Field
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -139,25 +136,6 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
   throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
 }
 
-
-final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
-final String minioBinDir = "${buildDir}/minio/bin"
-final String minioDataDir = "${buildDir}/minio/data"
-final String minioAddress = "127.0.0.1"
-
-String minioDistribution
-String minioCheckSum
-if (Os.isFamily(Os.FAMILY_MAC)) {
-  minioDistribution = 'darwin-amd64'
-  minioCheckSum = '96b0bcb2f590e8e65fb83d5c3e221f9bd1106b49fa6f22c6b726b80b845d7c60'
-} else if (Os.isFamily(Os.FAMILY_UNIX)) {
-  minioDistribution = 'linux-amd64'
-  minioCheckSum = '713dac7c105285eab3b92649be92b5e793b29d3525c7929fa7aaed99374fad99'
-} else {
-  minioDistribution = null
-  minioCheckSum = null
-}
-
 buildscript {
   repositories {
     maven {
@@ -169,164 +147,69 @@ buildscript {
   }
 }
 
-if (useFixture && minioDistribution) {
-  apply plugin: 'de.undercouch.download'
-
-  final String minioFileName = "minio.${minioVersion}"
-  final String minioDownloadURL = "https://dl.minio.io/server/minio/release/${minioDistribution}/archive/${minioFileName}"
-  final String minioFilePath = "${gradle.gradleUserHomeDir}/downloads/minio/${minioDistribution}/${minioFileName}"
-
-  task downloadMinio(type: Download) {
-    src minioDownloadURL
-    dest minioFilePath
-    onlyIfModified true
-  }
+if (useFixture) {
 
-  task verifyMinioChecksum(type: Verify, dependsOn: downloadMinio) {
-    src minioFilePath
-    algorithm 'SHA-256'
-    checksum minioCheckSum
-  }
+  apply plugin: 'elasticsearch.test.fixtures'
 
-  task installMinio(type: Sync, dependsOn: verifyMinioChecksum) {
-    from minioFilePath
-    into minioBinDir
-    fileMode 0755
+  RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) {
+    description = "Runs REST tests using the Minio repository."
   }
 
-  task startMinio {
-    dependsOn installMinio
-
-    ext.minioPid = 0L
-    ext.minioPort = 0
-
+  Task writeDockerFile = project.tasks.create('writeDockerFile') {
+    File minioDockerfile = new File("${project.buildDir}/minio-docker/Dockerfile")
+    outputs.file(minioDockerfile)
     doLast {
-      // get free port
-      for (int port = 60920; port < 60940; port++) {
-        try {
-          javax.net.ServerSocketFactory.getDefault().createServerSocket(port, 1, InetAddress.getByName(minioAddress)).close()
-          minioPort = port
-          break
-        } catch (BindException e) {
-          logger.info("Port " + port + " for Minio process is already taken", e)
-        }
-      }
-      if (minioPort == 0) {
-        throw new GradleException("Could not find a free port for Minio")
-      }
-
-      new File("${minioDataDir}/${s3PermanentBucket}").mkdirs()
-      // we skip these tests on Windows so we do no need to worry about compatibility here
-      final ProcessBuilder minio = new ProcessBuilder(
-              "${minioBinDir}/${minioFileName}",
-              "server",
-              "--address",
-              minioAddress + ":" + minioPort,
-              minioDataDir)
-      minio.environment().put('MINIO_ACCESS_KEY', s3PermanentAccessKey)
-      minio.environment().put('MINIO_SECRET_KEY', s3PermanentSecretKey)
-      final Process process = minio.start()
-      if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
-        try {
-          Class<?> cProcessImpl = process.getClass()
-          Field fPid = cProcessImpl.getDeclaredField("pid")
-          if (!fPid.isAccessible()) {
-            fPid.setAccessible(true)
-          }
-          minioPid = fPid.getInt(process)
-        } catch (Exception e) {
-          logger.error("failed to read pid from minio process", e)
-          process.destroyForcibly()
-          throw e
-        }
-      } else {
-        minioPid = process.pid()
-      }
-
-      new BufferedReader(new InputStreamReader(process.getInputStream())).withReader { br ->
-        String line
-        int httpPort = 0
-        while ((line = br.readLine()) != null) {
-          logger.info(line)
-          if (line.matches('.*Endpoint.*:\\d+$')) {
-            assert httpPort == 0
-            final int index = line.lastIndexOf(":")
-            assert index >= 0
-            httpPort = Integer.parseInt(line.substring(index + 1))
-            assert httpPort == minioPort : "Port mismatch, expected ${minioPort} but was ${httpPort}"
-
-            final File script = new File(project.buildDir, "minio/minio.killer.sh")
-            script.setText(
-                    ["function shutdown {",
-                     "  kill ${minioPid}",
-                     "}",
-                     "trap shutdown EXIT",
-                     // will wait indefinitely for input, but we never pass input, and the pipe is only closed when the build dies
-                     "read line\n"].join('\n'), 'UTF-8')
-            final ProcessBuilder killer = new ProcessBuilder("bash", script.absolutePath)
-            killer.start()
-            break
-          }
-        }
-
-        assert httpPort > 0
-      }
-    }
-  }
-
-  task stopMinio(type: LoggedExec) {
-    onlyIf { startMinio.minioPid > 0 }
-
-    doFirst {
-      logger.info("Shutting down minio with pid ${startMinio.minioPid}")
+      minioDockerfile.parentFile.mkdirs()
+      minioDockerfile.text = "FROM minio/minio:RELEASE.2019-01-23T23-18-58Z\n" +
+        "RUN mkdir -p /minio/data/${s3PermanentBucket}\n" +
+        "ENV MINIO_ACCESS_KEY ${s3PermanentAccessKey}\n" +
+        "ENV MINIO_SECRET_KEY ${s3PermanentSecretKey}"
     }
-
-    final Object pid = "${ -> startMinio.minioPid }"
-
-    // we skip these tests on Windows so we do no need to worry about compatibility here
-    executable = 'kill'
-    args('-9', pid)
   }
 
-  RestIntegTestTask integTestMinio = project.tasks.create('integTestMinio', RestIntegTestTask.class) {
-    description = "Runs REST tests using the Minio repository."
-  }
+  preProcessFixture.dependsOn(writeDockerFile)
+  composeUp.dependsOn(writeDockerFile)
 
   // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
   project.afterEvaluate {
-    ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration
-    cluster.dependsOn(project.bundlePlugin)
-    cluster.dependsOn(startMinio) // otherwise we don't know the Minio port
-    cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
-    cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
-
-    Closure<String> minioAddressAndPort = {
-      assert startMinio.minioPort > 0
-      return 'http://' + minioAddress + ':' + startMinio.minioPort
-    }
-    cluster.setting 's3.client.integration_test_permanent.endpoint', "${ -> minioAddressAndPort.call()}"
+    // Only configure the Minio tests if postProcessFixture is configured to skip them if Docker is not available
+    // or fixtures have been disabled
+    if (postProcessFixture.enabled) {
+      ClusterConfiguration cluster = project.extensions.getByName('integTestMinioCluster') as ClusterConfiguration
+      cluster.dependsOn(project.bundlePlugin)
+      cluster.dependsOn(postProcessFixture)
+      cluster.keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
+      cluster.keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
+
+      Closure<String> minioAddressAndPort = {
+        int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
+        assert minioPort > 0
+        return 'http://127.0.0.1:' + minioPort
+      }
+      cluster.setting 's3.client.integration_test_permanent.endpoint', "${-> minioAddressAndPort.call()}"
 
-    Task restIntegTestTask = project.tasks.getByName('integTestMinio')
-    restIntegTestTask.clusterConfig.plugin(project.path)
+      Task restIntegTestTask = project.tasks.getByName('integTestMinio')
+      restIntegTestTask.clusterConfig.plugin(project.path)
 
-    // Default jvm arguments for all test clusters
-    String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
-            " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
-            " " + System.getProperty('tests.jvm.argline', '')
+      // Default jvm arguments for all test clusters
+      String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
+        " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
+        " " + System.getProperty('tests.jvm.argline', '')
 
-    restIntegTestTask.clusterConfig.jvmArgs = jvmArgs
+      restIntegTestTask.clusterConfig.jvmArgs = jvmArgs
+      project.check.dependsOn(integTestMinio)
+    }
   }
 
-  integTestMinioRunner.dependsOn(startMinio)
-  integTestMinioRunner.finalizedBy(stopMinio)
+  integTestMinioRunner.dependsOn(postProcessFixture)
   // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
   integTestMinioRunner.systemProperty 'tests.rest.blacklist', [
           'repository_s3/30_repository_temporary_credentials/*',
           'repository_s3/40_repository_ec2_credentials/*',
           'repository_s3/50_repository_ecs_credentials/*'
         ].join(",")
 
-  project.check.dependsOn(integTestMinio)
+  BuildPlugin.requireDocker(integTestMinio)
 }
 
 File parentFixtures = new File(project.buildDir, "fixtures")

plugins/repository-s3/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3'
+services:
+  minio-fixture:
+    build:
+      context: ./build/minio-docker
+      dockerfile: Dockerfile
+    ports:
+      - "9000"
+    command: ["server", "/minio/data"]