Add docker-compose fixtures for S3 integration tests (#49107)

Similarly to what has been done for Azure (#48636) and GCS (#48762), 
this committ removes the existing Ant fixture that emulates a S3 storage 
service in favor of multiple docker-compose based fixtures.

The goals here are multiple: be able to reuse a s3-fixture outside of the 
repository-s3 plugin; allow parallel execution of integration tests; removes 
the existing AmazonS3Fixture that has evolved in a weird beast in 
dedicated, more maintainable fixtures.

The server side logic that emulates S3 mostly comes from the latest 
HttpHandler made for S3 blob store repository tests, with additional 
features extracted from the (now removed) AmazonS3Fixture: 
authentication checks, session token checks and improved response 
errors. Chunked upload request support for S3 object has been added 
too. 

The server side logic of all tests now reside in a single S3HttpHandler class.

Whereas AmazonS3Fixture contained logic for basic tests, session token 
tests, EC2 tests or ECS tests, the S3 fixtures are now dedicated to each 
kind of test. Fixtures are inheriting from each other, making things easier 
to maintain.

Author: tlrx

plugins/repository-s3/build.gradle

@@ -1,6 +1,5 @@
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.info.BuildParams
-import org.elasticsearch.gradle.test.AntFixture
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
 import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
@@ -23,6 +22,7 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.test.fixtures'
 
 esplugin {
   description 'The S3 repository plugin adds S3 repositories'
@@ -52,7 +52,7 @@ dependencies {
   // and whitelist this hack in JarHell
   compile 'javax.xml.bind:jaxb-api:2.2.2'
 
-  testCompile project(':test:fixtures:minio-fixture')
+  testCompile project(':test:fixtures:s3-fixture')
 }
 
 dependencyLicenses {
@@ -104,7 +104,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
   s3PermanentAccessKey = 'access_key'
   s3PermanentSecretKey = 'secret_key'
   s3PermanentBucket = 'bucket'
-  s3PermanentBasePath = ''
+  s3PermanentBasePath = 'base_path'
 
   useFixture = true
 
@@ -113,21 +113,21 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
 }
 
 if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
-  s3TemporaryAccessKey = 's3_integration_test_temporary_access_key'
-  s3TemporarySecretKey = 's3_integration_test_temporary_secret_key'
-  s3TemporaryBucket = 'temporary-bucket-test'
-  s3TemporaryBasePath = 'integration_test'
-  s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
+  s3TemporaryAccessKey = 'session_token_access_key'
+  s3TemporarySecretKey = 'session_token_secret_key'
+  s3TemporaryBucket = 'session_token_bucket'
+  s3TemporaryBasePath = 'session_token_base_path'
+  s3TemporarySessionToken = 'session_token'
 
 } else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
   throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
 }
 
 if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
-  s3EC2Bucket = 'ec2-bucket-test'
-  s3EC2BasePath = 'integration_test'
-  s3ECSBucket = 'ecs-bucket-test'
-  s3ECSBasePath = 'integration_test'
+  s3EC2Bucket = 'ec2_bucket'
+  s3EC2BasePath = 'ec2_base_path'
+  s3ECSBucket = 'ecs_bucket'
+  s3ECSBasePath = 'ecs_base_path'
 } else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
   throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
 }
@@ -141,8 +141,6 @@ task thirdPartyTest(type: Test) {
 }
 
 if (useFixture) {
-  apply plugin: 'elasticsearch.test.fixtures'
-
   testFixtures.useFixture(':test:fixtures:minio-fixture')
 
   def minioAddress = {
@@ -200,39 +198,6 @@ if (useFixture) {
 
 check.dependsOn(thirdPartyTest)
 
-File parentFixtures = new File(project.buildDir, "fixtures")
-File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')
-
-task s3FixtureProperties {
-  outputs.file(s3FixtureFile)
-  def s3FixtureOptions = [
-    "tests.seed"                       : BuildParams.testSeed,
-    "s3Fixture.permanent_bucket_name"  : s3PermanentBucket,
-    "s3Fixture.permanent_key"          : s3PermanentAccessKey,
-    "s3Fixture.temporary_bucket_name"  : s3TemporaryBucket,
-    "s3Fixture.temporary_key"          : s3TemporaryAccessKey,
-    "s3Fixture.temporary_session_token": s3TemporarySessionToken,
-    "s3Fixture.ec2_bucket_name"        : s3EC2Bucket,
-    "s3Fixture.ecs_bucket_name"        : s3ECSBucket,
-    "s3Fixture.disableChunkedEncoding" : s3DisableChunkedEncoding
-  ]
-
-  doLast {
-    file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n")
-  }
-}
-
-/** A task to start the AmazonS3Fixture which emulates an S3 service **/
-task s3Fixture(type: AntFixture) {
-  dependsOn testClasses
-  dependsOn s3FixtureProperties
-  inputs.file(s3FixtureFile)
-
-  env 'CLASSPATH', "${-> project.sourceSets.test.runtimeClasspath.asPath}"
-  executable = "${BuildParams.runtimeJavaHome}/bin/java"
-  args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3FixtureFile.getAbsolutePath()
-}
-
 processTestResources {
   Map<String, Object> expansions = [
     'permanent_bucket'        : s3PermanentBucket,
@@ -249,8 +214,13 @@ processTestResources {
   MavenFilteringHack.filter(it, expansions)
 }
 
-integTest {
-  dependsOn s3Fixture
+testFixtures.useFixture(':test:fixtures:s3-fixture')
+
+def fixtureAddress = { fixture ->
+  assert useFixture: 'closure should not be used without a fixture'
+  int ephemeralPort = project(':test:fixtures:s3-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
+  assert ephemeralPort > 0
+  'http://127.0.0.1:' + ephemeralPort
 }
 
 testClusters.integTest {
@@ -262,12 +232,12 @@ testClusters.integTest {
   keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
 
   if (useFixture) {
-    setting 's3.client.integration_test_permanent.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
-    setting 's3.client.integration_test_temporary.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
-    setting 's3.client.integration_test_ec2.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
+    setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture')}" }, IGNORE_VALUE
+    setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture-with-session-token')}" }, IGNORE_VALUE
+    setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
 
     // to redirect InstanceProfileCredentialsProvider to custom auth point
-    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
+    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
   } else {
     println "Using an external service to test the repository-s3 plugin"
   }
@@ -280,7 +250,7 @@ task s3ThirdPartyTests {
 if (useFixture) {
   task integTestECS(type: RestIntegTestTask.class) {
     description = "Runs tests using the ECS repository."
-    dependsOn(project.s3Fixture, 'bundlePlugin')
+    dependsOn('bundlePlugin')
     runner {
       systemProperty 'tests.rest.blacklist', [
         'repository_s3/10_basic/*',
@@ -293,9 +263,9 @@ if (useFixture) {
   check.dependsOn(integTestECS)
 
   testClusters.integTestECS {
-    setting 's3.client.integration_test_ecs.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
+    setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE
     plugin file(tasks.bundlePlugin.archiveFile)
-    environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "http://${s3Fixture.addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE
+    environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE
   }
 
   gradle.taskGraph.whenReady {