From 66e9837c743f6f076ad328be0b60e41924052a15 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 18 May 2018 10:41:23 +0200 Subject: [PATCH 1/9] Update AWS SDK to 1.11.331 in repository-s3 This commit updates the AWS SDK for the repository-s3 plugin. --- plugins/repository-s3/build.gradle | 11 ++- .../aws-java-sdk-core-1.11.223.jar.sha1 | 1 - .../aws-java-sdk-core-1.11.331.jar.sha1 | 1 + .../aws-java-sdk-kms-1.11.223.jar.sha1 | 1 - .../aws-java-sdk-kms-1.11.331.jar.sha1 | 1 + .../aws-java-sdk-s3-1.11.223.jar.sha1 | 1 - .../aws-java-sdk-s3-1.11.331.jar.sha1 | 1 + .../jackson-annotations-2.6.0.jar.sha1 | 1 - .../jackson-annotations-2.8.10.jar.sha1 | 1 + .../jackson-databind-2.6.7.1.jar.sha1 | 1 - .../licenses/jackson-databind-2.8.10.jar.sha1 | 1 + .../licenses/jmespath-java-1.11.331.jar.sha1 | 1 + .../test/repository_s3/10_repository.yml | 14 ++++ .../repositories/s3/InternalAwsS3Service.java | 74 +++++++++++++++---- .../repositories/s3/S3BlobStore.java | 9 +-- .../repositories/s3/S3Repository.java | 2 - .../s3/AwsS3ServiceImplTests.java | 61 ++++++++++++++- .../repositories/s3/MockAmazonS3.java | 3 +- .../repositories/s3/S3RepositoryTests.java | 2 +- 19 files changed, 151 insertions(+), 36 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.331.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.331.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.331.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.6.0.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.8.10.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.6.7.1.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.10.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.331.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 23252881cd75f..1c57d07b5d92e 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.11.223' + 'aws': '1.11.331' ] dependencies { @@ -34,16 +34,19 @@ dependencies { compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" compile "commons-logging:commons-logging:${versions.commonslogging}" compile "commons-codec:commons-codec:${versions.commonscodec}" - compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1' - compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0' + compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + compile "com.amazonaws:jmespath-java:${versions.aws}" // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here, - // and whitelist this hack in JarHell + // and whitelist this hack in JarHell compile 'javax.xml.bind:jaxb-api:2.2.2' } dependencyLicenses { mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk' + mapping from: /jmespath-java.*/, to: 'aws-java-sdk' mapping from: /jackson-.*/, to: 'jackson' mapping from: /jaxb-.*/, to: 'jaxb' } diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 deleted file mode 100644 index 9890dd8d600b3..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c3993cb44f5856fa721b7b7ccfc266377c0bf9c0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.331.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.331.jar.sha1 new file mode 100644 index 0000000000000..fe2bac8128d76 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.331.jar.sha1 @@ -0,0 +1 @@ +83e63032e9f873e0691748489522a5bc6640cf98 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 deleted file mode 100644 index d5bc9d30308dc..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c24e6ebe108c60a08098aeaad5ae0b6a5a77b618 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.331.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.331.jar.sha1 new file mode 100644 index 0000000000000..dc64e9c4921ee --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.331.jar.sha1 @@ -0,0 +1 @@ +5e5dc511ebcf2b2b042e26dfdcf48db60ba20211 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 deleted file mode 100644 index fe12b2d4847fa..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c2ef96732e22d97952fbcd0a94f1dc376d157eda \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.331.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.331.jar.sha1 new file mode 100644 index 0000000000000..58bf4b2cfacf0 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.331.jar.sha1 @@ -0,0 +1 @@ +868ae2f7c339a6ead87a6f21f7b3aa8b6c599153 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.6.0.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.6.0.jar.sha1 deleted file mode 100644 index bc4cae402d631..0000000000000 --- a/plugins/repository-s3/licenses/jackson-annotations-2.6.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a0990e2e812ac6639b6ce955c91b13228500476e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.8.10.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.8.10.jar.sha1 new file mode 100644 index 0000000000000..16addfe35076c --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-annotations-2.8.10.jar.sha1 @@ -0,0 +1 @@ +5e924646d6f893bc9036939c5f2b4ecaee85e5da \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.6.7.1.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.6.7.1.jar.sha1 deleted file mode 100644 index 7d82dbddc52d0..0000000000000 --- a/plugins/repository-s3/licenses/jackson-databind-2.6.7.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -306775aeb5164835a1dcbdf3f945587045cfb3b5 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.10.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.10.jar.sha1 new file mode 100644 index 0000000000000..f8c5539424553 --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-databind-2.8.10.jar.sha1 @@ -0,0 +1 @@ +f7b83cb2bc4b88d53961e749e1ad32f49ef017b7 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.331.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.331.jar.sha1 new file mode 100644 index 0000000000000..23b7adec09f2c --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.331.jar.sha1 @@ -0,0 +1 @@ +84412c87699b2e74ade056cce76ad8d8e513795a \ No newline at end of file diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml index 8b3daccf0a2d7..00ab57ac027c9 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml @@ -181,3 +181,17 @@ - do: snapshot.delete_repository: repository: repository + +--- +"Bucket does not exist": + - do: + catch: /exception/ + snapshot.create_repository: + repository: repository + body: + type: s3 + settings: + bucket: does_not_exist + client: integration_test + base_path: ${base_path} + storage_class: onezone_ia diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index d70ed9ea9aa8b..b72da450954a7 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -24,10 +24,13 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.InstanceProfileCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.internal.Constants; +import com.amazonaws.util.AwsHostNameUtils; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; @@ -41,6 +44,8 @@ import java.util.Map; import java.util.function.Function; +import static com.amazonaws.services.s3.AmazonS3Client.S3_SERVICE_NAME; + class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service { @@ -49,7 +54,7 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se private final Map clientsSettings; - private final Map clientsCache = new HashMap<>(); + private final Map clientsCache = new HashMap<>(); InternalAwsS3Service(Settings settings, Map clientsSettings) { super(settings); @@ -58,35 +63,47 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se @Override public synchronized AmazonS3 client(Settings repositorySettings) { - String clientName = CLIENT_NAME.get(repositorySettings); - AmazonS3Client client = clientsCache.get(clientName); + final String clientName = CLIENT_NAME.get(repositorySettings); + AmazonS3 client = clientsCache.get(clientName); if (client != null) { return client; } - S3ClientSettings clientSettings = clientsSettings.get(clientName); + final S3ClientSettings clientSettings = clientsSettings.get(clientName); if (clientSettings == null) { throw new IllegalArgumentException("Unknown s3 client name [" + clientName + "]. Existing client configs: " + Strings.collectionToDelimitedString(clientsSettings.keySet(), ",")); } - logger.debug("creating S3 client with client_name [{}], endpoint [{}]", clientName, clientSettings.endpoint); + final AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard(); + logger.debug("creating client with name [{}] and endpoint [{}]", clientName, clientSettings.endpoint); - AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, clientSettings, repositorySettings); - ClientConfiguration configuration = buildConfiguration(clientSettings); + final AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, clientSettings, repositorySettings); + clientBuilder.withCredentials(credentials); - client = new AmazonS3Client(credentials, configuration); + final ClientConfiguration configuration = buildConfiguration(clientSettings); + clientBuilder.withClientConfiguration(configuration); - if (Strings.hasText(clientSettings.endpoint)) { - client.setEndpoint(clientSettings.endpoint); + final EndpointConfiguration endpoint = buildEndpoint(clientSettings); + clientBuilder.withEndpointConfiguration(endpoint); + if (Constants.S3_HOSTNAME.equals(endpoint.getServiceEndpoint())) { + // Enable global bucket access when using the default AWS endpoint. We rely on the AWS SDK to + // execute the first HTTP request using the default endpoint and to catch the redirect HTTP + // response. This response contains a "x-amz-bucket-region" header which indicates the region + // of the bucket. The AWS SDK also keeps an internal bucket/region cache. + clientBuilder.enableForceGlobalBucketAccess(); } + client = clientBuilder.build(); + logger.debug("client [{}] created with service endpoint [{}] and signing region [{}]", + clientName, endpoint.getServiceEndpoint(), endpoint.getSigningRegion()); + clientsCache.put(clientName, client); return client; } // pkg private for tests - static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { + static ClientConfiguration buildConfiguration(final S3ClientSettings clientSettings) { ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. @@ -112,7 +129,6 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger, S3ClientSettings clientSettings, Settings repositorySettings) { - BasicAWSCredentials credentials = clientSettings.credentials; if (S3Repository.ACCESS_KEY_SETTING.exists(repositorySettings)) { if (S3Repository.SECRET_KEY_SETTING.exists(repositorySettings) == false) { @@ -139,6 +155,36 @@ static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger } } + /** + * Builds a {@link EndpointConfiguration} from the client settings. + * + * When no endpoint is defined in the client settings, the default "s3.amazonaws.com" endpoint is used with the + * default region provided by the AWS SDK. We rely on the AWS SDK and {@link AmazonS3ClientBuilder#enableForceGlobalBucketAccess()} + * to later change this default region to the appropriate one. + * + * When the endpoint is defined in the client settings, the region is derived from the endpoint using the AWS SDK + * if the endpoint is a known Amazon S3 endpoint. If it's not an Amazon S3 endpoint, then the region is null. + * + * @param clientSettings the client settings + * @return a new {@link EndpointConfiguration} + */ + static EndpointConfiguration buildEndpoint(final S3ClientSettings clientSettings) { + final String serviceEndpoint = Strings.hasText(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME; + + String signingRegion = null; + if (isAwsStandardEndpoint(serviceEndpoint)) { + signingRegion = AwsHostNameUtils.parseRegion(serviceEndpoint, S3_SERVICE_NAME); + if (signingRegion == null) { + throw new IllegalArgumentException("Unable to find region for endpoint [" + serviceEndpoint + "]"); + } + } + return new EndpointConfiguration(serviceEndpoint, signingRegion); + } + + static boolean isAwsStandardEndpoint(final String endpoint) { + return endpoint.endsWith(".amazonaws.com") || endpoint.endsWith(".amazonaws.com.cn"); + } + @Override protected void doStart() throws ElasticsearchException { } @@ -149,7 +195,7 @@ protected void doStop() throws ElasticsearchException { @Override protected void doClose() throws ElasticsearchException { - for (AmazonS3Client client : clientsCache.values()) { + for (AmazonS3 client : clientsCache.values()) { client.shutdown(); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 27349f12135ed..1c076953b12a7 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -63,15 +63,10 @@ class S3BlobStore extends AbstractComponent implements BlobStore { this.cannedACL = initCannedACL(cannedACL); this.storageClass = initStorageClass(storageClass); - // Note: the method client.doesBucketExist() may return 'true' is the bucket exists - // but we don't have access to it (ie, 403 Forbidden response code) - // Also, if invalid security credentials are used to execute this method, the - // client is not able to distinguish between bucket permission errors and - // invalid credential errors, and this method could return an incorrect result. SocketAccess.doPrivilegedVoid(() -> { - if (client.doesBucketExist(bucket) == false) { + if (client.doesBucketExistV2(bucket) == false) { throw new IllegalArgumentException("The bucket [" + bucket + "] does not exist. Please create it before " + - " creating an s3 snapshot repository backed by it."); + " creating a s3 snapshot repository backed by it."); } }); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index cb4f977bae77b..26feeee6d489e 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -35,8 +35,6 @@ import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; -import java.io.IOException; - /** * Shared file system implementation of the BlobStoreRepository *

diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java index 18c701f5fc1a6..6ea15741addc5 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java @@ -23,15 +23,23 @@ import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.SecureSetting; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matcher; +import java.util.Locale; +import java.util.Random; + +import static org.elasticsearch.repositories.s3.S3ClientSettings.ENDPOINT_SETTING; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isEmptyOrNullString; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; public class AwsS3ServiceImplTests extends ESTestCase { @@ -166,4 +174,53 @@ private void assertEndpoint(Settings repositorySettings, Settings settings, Stri assertThat(clientSettings.endpoint, is(expectedEndpoint)); } + public void testIsAwsStandardEndpoint() { + final String standardEnpoint = randomStandardEndpoint(random()); + assertThat(InternalAwsS3Service.isAwsStandardEndpoint(standardEnpoint), is(true)); + + final String customEnpoint = randomFrom("http://s3.elastic.co", "https://storage.elastic.co/"); + assertThat(InternalAwsS3Service.isAwsStandardEndpoint(customEnpoint), is(false)); + } + + public void testBuildEndpoint() { + asserEndpointConfiguration(null, equalTo("s3.amazonaws.com"), not(isEmptyOrNullString())); + asserEndpointConfiguration("", equalTo("s3.amazonaws.com"), not(isEmptyOrNullString())); + asserEndpointConfiguration("s3.amazonaws.com", equalTo("s3.amazonaws.com"), not(isEmptyOrNullString())); + asserEndpointConfiguration("test.amazonaws.com", equalTo("test.amazonaws.com"), not(isEmptyOrNullString())); + asserEndpointConfiguration("s3.us-west-1.amazonaws.com", equalTo("s3.us-west-1.amazonaws.com"), equalTo("us-west-1")); + asserEndpointConfiguration("s3.cn-north-1.amazonaws.com.cn", equalTo("s3.cn-north-1.amazonaws.com.cn"), equalTo("cn-north-1")); + asserEndpointConfiguration("service.region.amazonaws.com", equalTo("service.region.amazonaws.com"), equalTo("region")); + asserEndpointConfiguration("http://s3.elastic.co", equalTo("http://s3.elastic.co"), is(nullValue())); + asserEndpointConfiguration("https://storage.elastic.co:443", equalTo("https://storage.elastic.co:443"), is(nullValue())); + } + + private static String randomStandardEndpoint(final Random random) { + final StringBuilder endpoint = new StringBuilder(); + endpoint.append(randomFrom("s3", "service", "test", "my-bucket")); + if (frequently()) { + endpoint.append(randomFrom("-", ".")); + endpoint.append(randomFrom("us-east-1", "us-west-1", "ca-central-1", "eu-west-3")); + } + endpoint.append(".amazonaws.com"); + if (rarely()) { + endpoint.append(".cn"); + } + return endpoint.toString(); + } + + private static void asserEndpointConfiguration(final String endpoint, + final Matcher matcherEndpoint, + final Matcher matcherRegion) { + + final String clientName = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); + final Settings settings = Settings.builder() + .put(ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint) + .build(); + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, clientName); + final AwsClientBuilder.EndpointConfiguration configuration = InternalAwsS3Service.buildEndpoint(clientSettings); + + assertNotNull(configuration); + assertThat(configuration.getServiceEndpoint(), matcherEndpoint); + assertThat(configuration.getSigningRegion(), matcherRegion); + } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java index 33d5d5fbc2038..cbdab26f6c262 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java @@ -20,6 +20,7 @@ package org.elasticsearch.repositories.s3; import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AbstractAmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; @@ -73,7 +74,7 @@ class MockAmazonS3 extends AbstractAmazonS3 { } @Override - public boolean doesBucketExist(final String bucket) { + public boolean doesBucketExistV2(String bucketName) throws SdkClientException { return this.bucket.equalsIgnoreCase(bucket); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java index 7da65c27d8194..5ae714be04ddb 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java @@ -40,7 +40,7 @@ public class S3RepositoryTests extends ESTestCase { private static class DummyS3Client extends AbstractAmazonS3 { @Override - public boolean doesBucketExist(String bucketName) { + public boolean doesBucketExistV2(String bucketName) { return true; } } From a5deb39308079b32918d13fb62fd467422c2b29b Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 22 May 2018 10:05:38 +0200 Subject: [PATCH 2/9] Bump to 1.11.333 --- plugins/repository-s3/build.gradle | 2 +- .../repository-s3/licenses/aws-java-sdk-core-1.11.331.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-kms-1.11.331.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-s3-1.11.331.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 | 1 + plugins/repository-s3/licenses/jmespath-java-1.11.331.jar.sha1 | 1 - plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 | 1 + .../org/elasticsearch/repositories/s3/InternalAwsS3Service.java | 1 + 10 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.331.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.331.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.331.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.331.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 1c57d07b5d92e..037fc55f5c76c 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.11.331' + 'aws': '1.11.333' ] dependencies { diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.331.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.331.jar.sha1 deleted file mode 100644 index fe2bac8128d76..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.331.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -83e63032e9f873e0691748489522a5bc6640cf98 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 new file mode 100644 index 0000000000000..1194946dbfd26 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 @@ -0,0 +1 @@ +1174d5ffeaa54f32eb4a20073d2f6637ed28ba4c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.331.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.331.jar.sha1 deleted file mode 100644 index dc64e9c4921ee..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.331.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e5dc511ebcf2b2b042e26dfdcf48db60ba20211 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 new file mode 100644 index 0000000000000..7dcc19650e1e8 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 @@ -0,0 +1 @@ +e2aac2deed0ecff4a8c9860cb8c9fb26d37ded48 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.331.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.331.jar.sha1 deleted file mode 100644 index 58bf4b2cfacf0..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.331.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -868ae2f7c339a6ead87a6f21f7b3aa8b6c599153 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 new file mode 100644 index 0000000000000..010a78f4f9fbd --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 @@ -0,0 +1 @@ +f5efa89205a9890eeda58964bbcc90a7e2104cef \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.331.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.331.jar.sha1 deleted file mode 100644 index 23b7adec09f2c..0000000000000 --- a/plugins/repository-s3/licenses/jmespath-java-1.11.331.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -84412c87699b2e74ade056cce76ad8d8e513795a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 new file mode 100644 index 0000000000000..e0653a5cc1957 --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 @@ -0,0 +1 @@ +ed78ed29d8b1d9f3a7530fd650fad0624456d089 \ No newline at end of file diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index b72da450954a7..1e78a7983465e 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -178,6 +178,7 @@ static EndpointConfiguration buildEndpoint(final S3ClientSettings clientSettings throw new IllegalArgumentException("Unable to find region for endpoint [" + serviceEndpoint + "]"); } } + // Service endpoint here can be with or without protocol return new EndpointConfiguration(serviceEndpoint, signingRegion); } From 92f85ae9ad20e181a7748ab8aee54f501e6b5f0a Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 24 May 2018 14:12:47 +0200 Subject: [PATCH 3/9] 1.11.335 --- plugins/repository-s3/build.gradle | 2 +- .../repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 | 1 + plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 | 1 - plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 | 1 + 9 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 037fc55f5c76c..1ba5bbdf52fad 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.11.333' + 'aws': '1.11.335' ] dependencies { diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 deleted file mode 100644 index 1194946dbfd26..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.333.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1174d5ffeaa54f32eb4a20073d2f6637ed28ba4c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 new file mode 100644 index 0000000000000..c888a16aca6ea --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 @@ -0,0 +1 @@ +6158201600eef7ad0453a26e357d66f1d504d054 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 deleted file mode 100644 index 7dcc19650e1e8..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.333.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e2aac2deed0ecff4a8c9860cb8c9fb26d37ded48 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 new file mode 100644 index 0000000000000..5fdbbf660376b --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 @@ -0,0 +1 @@ +ae03f20a41d1bcd9a3733a002504effcb82d251d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 deleted file mode 100644 index 010a78f4f9fbd..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.333.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f5efa89205a9890eeda58964bbcc90a7e2104cef \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 new file mode 100644 index 0000000000000..0610c6ea27fd0 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 @@ -0,0 +1 @@ +bc3b6c70be8c1c643885940cf88bd44d12599ea8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 deleted file mode 100644 index e0653a5cc1957..0000000000000 --- a/plugins/repository-s3/licenses/jmespath-java-1.11.333.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ed78ed29d8b1d9f3a7530fd650fad0624456d089 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 new file mode 100644 index 0000000000000..7e96969dab1e8 --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 @@ -0,0 +1 @@ +d3a9a6ac58614396c9b8522dde6ae1f75671b7d7 \ No newline at end of file From 3f22ba36c474c9fc3cb4350226c586ee02864d38 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Mon, 28 May 2018 10:20:19 +0200 Subject: [PATCH 4/9] Bump to 1.11.337 --- plugins/repository-s3/build.gradle | 2 +- .../repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 | 1 + plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 | 1 - plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 | 1 + 9 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 1ba5bbdf52fad..0f17e8497d976 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.11.335' + 'aws': '1.11.337' ] dependencies { diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 deleted file mode 100644 index c888a16aca6ea..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.335.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6158201600eef7ad0453a26e357d66f1d504d054 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 new file mode 100644 index 0000000000000..0262368cf91dc --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 @@ -0,0 +1 @@ +11e9307717bd723c14a1ba2b9b9efb954228cbdd \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 deleted file mode 100644 index 5fdbbf660376b..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.335.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ae03f20a41d1bcd9a3733a002504effcb82d251d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 new file mode 100644 index 0000000000000..073a29cb53632 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 @@ -0,0 +1 @@ +ec9a65f7bce0e1f3ff2445cd447f540dd814c43c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 deleted file mode 100644 index 0610c6ea27fd0..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.335.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bc3b6c70be8c1c643885940cf88bd44d12599ea8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 new file mode 100644 index 0000000000000..b883ac18faf6b --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 @@ -0,0 +1 @@ +9a62840e0a14793ddf41b3a4f3ff188f002d814b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 deleted file mode 100644 index 7e96969dab1e8..0000000000000 --- a/plugins/repository-s3/licenses/jmespath-java-1.11.335.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d3a9a6ac58614396c9b8522dde6ae1f75671b7d7 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 new file mode 100644 index 0000000000000..523f06886a4a9 --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 @@ -0,0 +1 @@ +3ed2e775aaf011c8f512abaebd756724f28571a8 \ No newline at end of file From 88ad15e8f573a2f201c2257b5f9da5ed5d2a5918 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Mon, 4 Jun 2018 16:16:48 +0200 Subject: [PATCH 5/9] Bump to 1.11.340 --- plugins/repository-s3/build.gradle | 2 +- .../repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 | 1 + plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 | 1 - plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 | 1 + 9 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 0f17e8497d976..0a8e047d66653 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.11.337' + 'aws': '1.11.340' ] dependencies { diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 deleted file mode 100644 index 0262368cf91dc..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.337.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11e9307717bd723c14a1ba2b9b9efb954228cbdd \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 new file mode 100644 index 0000000000000..9250f378b7049 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 @@ -0,0 +1 @@ +4a75c2d22e618d3738356e6621965921c9d18c00 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 deleted file mode 100644 index 073a29cb53632..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.337.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec9a65f7bce0e1f3ff2445cd447f540dd814c43c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 new file mode 100644 index 0000000000000..8f2740a25d31f --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 @@ -0,0 +1 @@ +1f56fddabf838f4d4ffe44bbbffe977ca12c90d1 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 deleted file mode 100644 index b883ac18faf6b..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.337.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9a62840e0a14793ddf41b3a4f3ff188f002d814b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 new file mode 100644 index 0000000000000..b7ed3d059323d --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 @@ -0,0 +1 @@ +9a09d813ca6bac97c2b89f905ff5e43bc463a379 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 deleted file mode 100644 index 523f06886a4a9..0000000000000 --- a/plugins/repository-s3/licenses/jmespath-java-1.11.337.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3ed2e775aaf011c8f512abaebd756724f28571a8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 new file mode 100644 index 0000000000000..6d61c459d765d --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 @@ -0,0 +1 @@ +8425685021e02de595e52985736a1c174f99ede8 \ No newline at end of file From 9a841a668ebdf92188957f032c238955d7162da6 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Mon, 11 Jun 2018 09:45:17 +0200 Subject: [PATCH 6/9] Bump to 1.11.344 --- plugins/repository-s3/build.gradle | 2 +- .../repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-core-1.11.344.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-kms-1.11.344.jar.sha1 | 1 + .../repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 | 1 - .../repository-s3/licenses/aws-java-sdk-s3-1.11.344.jar.sha1 | 1 + plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 | 1 - plugins/repository-s3/licenses/jmespath-java-1.11.344.jar.sha1 | 1 + 9 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.344.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.344.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.344.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.344.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 0a8e047d66653..8b542cb3af0cc 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.11.340' + 'aws': '1.11.344' ] dependencies { diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 deleted file mode 100644 index 9250f378b7049..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.340.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4a75c2d22e618d3738356e6621965921c9d18c00 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.344.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.344.jar.sha1 new file mode 100644 index 0000000000000..8d34573c6b738 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.344.jar.sha1 @@ -0,0 +1 @@ +2fb543593ed2cb2f004498fed7680ff3c1ef75fc \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 deleted file mode 100644 index 8f2740a25d31f..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.340.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1f56fddabf838f4d4ffe44bbbffe977ca12c90d1 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.344.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.344.jar.sha1 new file mode 100644 index 0000000000000..b05db8e801bd8 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.344.jar.sha1 @@ -0,0 +1 @@ +1ee0d1b43fffcc6c8faff33f54acdcbf1bcd1588 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 deleted file mode 100644 index b7ed3d059323d..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.340.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9a09d813ca6bac97c2b89f905ff5e43bc463a379 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.344.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.344.jar.sha1 new file mode 100644 index 0000000000000..02be1db7a95ec --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.344.jar.sha1 @@ -0,0 +1 @@ +800ebe8cfdb1f62f6a6f5714f4cf665841122099 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 deleted file mode 100644 index 6d61c459d765d..0000000000000 --- a/plugins/repository-s3/licenses/jmespath-java-1.11.340.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8425685021e02de595e52985736a1c174f99ede8 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.344.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.344.jar.sha1 new file mode 100644 index 0000000000000..96e2c0c811eed --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.344.jar.sha1 @@ -0,0 +1 @@ +dfae247862ecd11248d52f25e4ebe9c6a9ca577d \ No newline at end of file From d1e5c1479e123c545042861fb756817c3e4ff786 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Mon, 20 Aug 2018 09:59:04 +0200 Subject: [PATCH 7/9] Revert changes --- plugins/repository-s3/build.gradle | 11 +-- .../aws-java-sdk-core-1.11.223.jar.sha1 | 1 + .../aws-java-sdk-core-1.11.344.jar.sha1 | 1 - .../aws-java-sdk-kms-1.11.223.jar.sha1 | 1 + .../aws-java-sdk-kms-1.11.344.jar.sha1 | 1 - .../aws-java-sdk-s3-1.11.223.jar.sha1 | 1 + .../aws-java-sdk-s3-1.11.344.jar.sha1 | 1 - .../jackson-annotations-2.6.0.jar.sha1 | 1 + .../jackson-annotations-2.8.10.jar.sha1 | 1 - .../jackson-databind-2.6.7.1.jar.sha1 | 1 + .../licenses/jackson-databind-2.8.10.jar.sha1 | 1 - .../licenses/jmespath-java-1.11.344.jar.sha1 | 1 - .../test/repository_s3/10_repository.yml | 14 ---- .../repositories/s3/InternalAwsS3Service.java | 75 ++++--------------- .../repositories/s3/S3BlobStore.java | 9 ++- .../repositories/s3/S3Repository.java | 2 + .../s3/AwsS3ServiceImplTests.java | 61 +-------------- .../repositories/s3/MockAmazonS3.java | 3 +- .../repositories/s3/S3RepositoryTests.java | 2 +- 19 files changed, 36 insertions(+), 152 deletions(-) create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.344.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.344.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.344.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.6.0.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.8.10.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.6.7.1.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.8.10.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.344.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 8b542cb3af0cc..23252881cd75f 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -23,7 +23,7 @@ esplugin { } versions << [ - 'aws': '1.11.344' + 'aws': '1.11.223' ] dependencies { @@ -34,19 +34,16 @@ dependencies { compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" compile "commons-logging:commons-logging:${versions.commonslogging}" compile "commons-codec:commons-codec:${versions.commonscodec}" - compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" - compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" - compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - compile "com.amazonaws:jmespath-java:${versions.aws}" + compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1' + compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0' // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here, - // and whitelist this hack in JarHell + // and whitelist this hack in JarHell compile 'javax.xml.bind:jaxb-api:2.2.2' } dependencyLicenses { mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk' - mapping from: /jmespath-java.*/, to: 'aws-java-sdk' mapping from: /jackson-.*/, to: 'jackson' mapping from: /jaxb-.*/, to: 'jaxb' } diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 new file mode 100644 index 0000000000000..9890dd8d600b3 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 @@ -0,0 +1 @@ +c3993cb44f5856fa721b7b7ccfc266377c0bf9c0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.344.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.344.jar.sha1 deleted file mode 100644 index 8d34573c6b738..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.344.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fb543593ed2cb2f004498fed7680ff3c1ef75fc \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 new file mode 100644 index 0000000000000..d5bc9d30308dc --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 @@ -0,0 +1 @@ +c24e6ebe108c60a08098aeaad5ae0b6a5a77b618 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.344.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.344.jar.sha1 deleted file mode 100644 index b05db8e801bd8..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.344.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1ee0d1b43fffcc6c8faff33f54acdcbf1bcd1588 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 new file mode 100644 index 0000000000000..fe12b2d4847fa --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 @@ -0,0 +1 @@ +c2ef96732e22d97952fbcd0a94f1dc376d157eda \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.344.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.344.jar.sha1 deleted file mode 100644 index 02be1db7a95ec..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.344.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -800ebe8cfdb1f62f6a6f5714f4cf665841122099 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.6.0.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.6.0.jar.sha1 new file mode 100644 index 0000000000000..bc4cae402d631 --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-annotations-2.6.0.jar.sha1 @@ -0,0 +1 @@ +a0990e2e812ac6639b6ce955c91b13228500476e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.8.10.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.8.10.jar.sha1 deleted file mode 100644 index 16addfe35076c..0000000000000 --- a/plugins/repository-s3/licenses/jackson-annotations-2.8.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e924646d6f893bc9036939c5f2b4ecaee85e5da \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.6.7.1.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.6.7.1.jar.sha1 new file mode 100644 index 0000000000000..7d82dbddc52d0 --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-databind-2.6.7.1.jar.sha1 @@ -0,0 +1 @@ +306775aeb5164835a1dcbdf3f945587045cfb3b5 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.8.10.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.8.10.jar.sha1 deleted file mode 100644 index f8c5539424553..0000000000000 --- a/plugins/repository-s3/licenses/jackson-databind-2.8.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f7b83cb2bc4b88d53961e749e1ad32f49ef017b7 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.344.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.344.jar.sha1 deleted file mode 100644 index 96e2c0c811eed..0000000000000 --- a/plugins/repository-s3/licenses/jmespath-java-1.11.344.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dfae247862ecd11248d52f25e4ebe9c6a9ca577d \ No newline at end of file diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml index 00ab57ac027c9..8b3daccf0a2d7 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml @@ -181,17 +181,3 @@ - do: snapshot.delete_repository: repository: repository - ---- -"Bucket does not exist": - - do: - catch: /exception/ - snapshot.create_repository: - repository: repository - body: - type: s3 - settings: - bucket: does_not_exist - client: integration_test - base_path: ${base_path} - storage_class: onezone_ia diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java index 1e78a7983465e..d70ed9ea9aa8b 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/InternalAwsS3Service.java @@ -24,13 +24,10 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.InstanceProfileCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.internal.Constants; -import com.amazonaws.util.AwsHostNameUtils; +import com.amazonaws.services.s3.AmazonS3Client; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; @@ -44,8 +41,6 @@ import java.util.Map; import java.util.function.Function; -import static com.amazonaws.services.s3.AmazonS3Client.S3_SERVICE_NAME; - class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service { @@ -54,7 +49,7 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se private final Map clientsSettings; - private final Map clientsCache = new HashMap<>(); + private final Map clientsCache = new HashMap<>(); InternalAwsS3Service(Settings settings, Map clientsSettings) { super(settings); @@ -63,47 +58,35 @@ class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Se @Override public synchronized AmazonS3 client(Settings repositorySettings) { - final String clientName = CLIENT_NAME.get(repositorySettings); - AmazonS3 client = clientsCache.get(clientName); + String clientName = CLIENT_NAME.get(repositorySettings); + AmazonS3Client client = clientsCache.get(clientName); if (client != null) { return client; } - final S3ClientSettings clientSettings = clientsSettings.get(clientName); + S3ClientSettings clientSettings = clientsSettings.get(clientName); if (clientSettings == null) { throw new IllegalArgumentException("Unknown s3 client name [" + clientName + "]. Existing client configs: " + Strings.collectionToDelimitedString(clientsSettings.keySet(), ",")); } - final AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard(); - logger.debug("creating client with name [{}] and endpoint [{}]", clientName, clientSettings.endpoint); + logger.debug("creating S3 client with client_name [{}], endpoint [{}]", clientName, clientSettings.endpoint); - final AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, clientSettings, repositorySettings); - clientBuilder.withCredentials(credentials); + AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, clientSettings, repositorySettings); + ClientConfiguration configuration = buildConfiguration(clientSettings); - final ClientConfiguration configuration = buildConfiguration(clientSettings); - clientBuilder.withClientConfiguration(configuration); + client = new AmazonS3Client(credentials, configuration); - final EndpointConfiguration endpoint = buildEndpoint(clientSettings); - clientBuilder.withEndpointConfiguration(endpoint); - if (Constants.S3_HOSTNAME.equals(endpoint.getServiceEndpoint())) { - // Enable global bucket access when using the default AWS endpoint. We rely on the AWS SDK to - // execute the first HTTP request using the default endpoint and to catch the redirect HTTP - // response. This response contains a "x-amz-bucket-region" header which indicates the region - // of the bucket. The AWS SDK also keeps an internal bucket/region cache. - clientBuilder.enableForceGlobalBucketAccess(); + if (Strings.hasText(clientSettings.endpoint)) { + client.setEndpoint(clientSettings.endpoint); } - client = clientBuilder.build(); - logger.debug("client [{}] created with service endpoint [{}] and signing region [{}]", - clientName, endpoint.getServiceEndpoint(), endpoint.getSigningRegion()); - clientsCache.put(clientName, client); return client; } // pkg private for tests - static ClientConfiguration buildConfiguration(final S3ClientSettings clientSettings) { + static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. @@ -129,6 +112,7 @@ static ClientConfiguration buildConfiguration(final S3ClientSettings clientSetti static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger, S3ClientSettings clientSettings, Settings repositorySettings) { + BasicAWSCredentials credentials = clientSettings.credentials; if (S3Repository.ACCESS_KEY_SETTING.exists(repositorySettings)) { if (S3Repository.SECRET_KEY_SETTING.exists(repositorySettings) == false) { @@ -155,37 +139,6 @@ static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger } } - /** - * Builds a {@link EndpointConfiguration} from the client settings. - * - * When no endpoint is defined in the client settings, the default "s3.amazonaws.com" endpoint is used with the - * default region provided by the AWS SDK. We rely on the AWS SDK and {@link AmazonS3ClientBuilder#enableForceGlobalBucketAccess()} - * to later change this default region to the appropriate one. - * - * When the endpoint is defined in the client settings, the region is derived from the endpoint using the AWS SDK - * if the endpoint is a known Amazon S3 endpoint. If it's not an Amazon S3 endpoint, then the region is null. - * - * @param clientSettings the client settings - * @return a new {@link EndpointConfiguration} - */ - static EndpointConfiguration buildEndpoint(final S3ClientSettings clientSettings) { - final String serviceEndpoint = Strings.hasText(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME; - - String signingRegion = null; - if (isAwsStandardEndpoint(serviceEndpoint)) { - signingRegion = AwsHostNameUtils.parseRegion(serviceEndpoint, S3_SERVICE_NAME); - if (signingRegion == null) { - throw new IllegalArgumentException("Unable to find region for endpoint [" + serviceEndpoint + "]"); - } - } - // Service endpoint here can be with or without protocol - return new EndpointConfiguration(serviceEndpoint, signingRegion); - } - - static boolean isAwsStandardEndpoint(final String endpoint) { - return endpoint.endsWith(".amazonaws.com") || endpoint.endsWith(".amazonaws.com.cn"); - } - @Override protected void doStart() throws ElasticsearchException { } @@ -196,7 +149,7 @@ protected void doStop() throws ElasticsearchException { @Override protected void doClose() throws ElasticsearchException { - for (AmazonS3 client : clientsCache.values()) { + for (AmazonS3Client client : clientsCache.values()) { client.shutdown(); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 1c076953b12a7..27349f12135ed 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -63,10 +63,15 @@ class S3BlobStore extends AbstractComponent implements BlobStore { this.cannedACL = initCannedACL(cannedACL); this.storageClass = initStorageClass(storageClass); + // Note: the method client.doesBucketExist() may return 'true' is the bucket exists + // but we don't have access to it (ie, 403 Forbidden response code) + // Also, if invalid security credentials are used to execute this method, the + // client is not able to distinguish between bucket permission errors and + // invalid credential errors, and this method could return an incorrect result. SocketAccess.doPrivilegedVoid(() -> { - if (client.doesBucketExistV2(bucket) == false) { + if (client.doesBucketExist(bucket) == false) { throw new IllegalArgumentException("The bucket [" + bucket + "] does not exist. Please create it before " + - " creating a s3 snapshot repository backed by it."); + " creating an s3 snapshot repository backed by it."); } }); } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 26feeee6d489e..cb4f977bae77b 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -35,6 +35,8 @@ import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import java.io.IOException; + /** * Shared file system implementation of the BlobStoreRepository *

diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java index 6ea15741addc5..18c701f5fc1a6 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java @@ -23,23 +23,15 @@ import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder; import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.SecureSetting; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; -import org.hamcrest.Matcher; -import java.util.Locale; -import java.util.Random; - -import static org.elasticsearch.repositories.s3.S3ClientSettings.ENDPOINT_SETTING; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.isEmptyOrNullString; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.nullValue; public class AwsS3ServiceImplTests extends ESTestCase { @@ -174,53 +166,4 @@ private void assertEndpoint(Settings repositorySettings, Settings settings, Stri assertThat(clientSettings.endpoint, is(expectedEndpoint)); } - public void testIsAwsStandardEndpoint() { - final String standardEnpoint = randomStandardEndpoint(random()); - assertThat(InternalAwsS3Service.isAwsStandardEndpoint(standardEnpoint), is(true)); - - final String customEnpoint = randomFrom("http://s3.elastic.co", "https://storage.elastic.co/"); - assertThat(InternalAwsS3Service.isAwsStandardEndpoint(customEnpoint), is(false)); - } - - public void testBuildEndpoint() { - asserEndpointConfiguration(null, equalTo("s3.amazonaws.com"), not(isEmptyOrNullString())); - asserEndpointConfiguration("", equalTo("s3.amazonaws.com"), not(isEmptyOrNullString())); - asserEndpointConfiguration("s3.amazonaws.com", equalTo("s3.amazonaws.com"), not(isEmptyOrNullString())); - asserEndpointConfiguration("test.amazonaws.com", equalTo("test.amazonaws.com"), not(isEmptyOrNullString())); - asserEndpointConfiguration("s3.us-west-1.amazonaws.com", equalTo("s3.us-west-1.amazonaws.com"), equalTo("us-west-1")); - asserEndpointConfiguration("s3.cn-north-1.amazonaws.com.cn", equalTo("s3.cn-north-1.amazonaws.com.cn"), equalTo("cn-north-1")); - asserEndpointConfiguration("service.region.amazonaws.com", equalTo("service.region.amazonaws.com"), equalTo("region")); - asserEndpointConfiguration("http://s3.elastic.co", equalTo("http://s3.elastic.co"), is(nullValue())); - asserEndpointConfiguration("https://storage.elastic.co:443", equalTo("https://storage.elastic.co:443"), is(nullValue())); - } - - private static String randomStandardEndpoint(final Random random) { - final StringBuilder endpoint = new StringBuilder(); - endpoint.append(randomFrom("s3", "service", "test", "my-bucket")); - if (frequently()) { - endpoint.append(randomFrom("-", ".")); - endpoint.append(randomFrom("us-east-1", "us-west-1", "ca-central-1", "eu-west-3")); - } - endpoint.append(".amazonaws.com"); - if (rarely()) { - endpoint.append(".cn"); - } - return endpoint.toString(); - } - - private static void asserEndpointConfiguration(final String endpoint, - final Matcher matcherEndpoint, - final Matcher matcherRegion) { - - final String clientName = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); - final Settings settings = Settings.builder() - .put(ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint) - .build(); - final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, clientName); - final AwsClientBuilder.EndpointConfiguration configuration = InternalAwsS3Service.buildEndpoint(clientSettings); - - assertNotNull(configuration); - assertThat(configuration.getServiceEndpoint(), matcherEndpoint); - assertThat(configuration.getSigningRegion(), matcherRegion); - } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java index cbdab26f6c262..33d5d5fbc2038 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java @@ -20,7 +20,6 @@ package org.elasticsearch.repositories.s3; import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AbstractAmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; @@ -74,7 +73,7 @@ class MockAmazonS3 extends AbstractAmazonS3 { } @Override - public boolean doesBucketExistV2(String bucketName) throws SdkClientException { + public boolean doesBucketExist(final String bucket) { return this.bucket.equalsIgnoreCase(bucket); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java index 5ae714be04ddb..7da65c27d8194 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java @@ -40,7 +40,7 @@ public class S3RepositoryTests extends ESTestCase { private static class DummyS3Client extends AbstractAmazonS3 { @Override - public boolean doesBucketExistV2(String bucketName) { + public boolean doesBucketExist(String bucketName) { return true; } } From 702060e80671633ba48ba328160468cca65911db Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Mon, 20 Aug 2018 11:00:35 +0200 Subject: [PATCH 8/9] Update AWS SDK to 1.11.390 --- plugins/repository-s3/build.gradle | 7 ++++- .../aws-java-sdk-core-1.11.223.jar.sha1 | 1 - .../aws-java-sdk-core-1.11.390.jar.sha1 | 1 + .../aws-java-sdk-kms-1.11.223.jar.sha1 | 1 - .../aws-java-sdk-kms-1.11.390.jar.sha1 | 1 + .../aws-java-sdk-s3-1.11.223.jar.sha1 | 1 - .../aws-java-sdk-s3-1.11.390.jar.sha1 | 1 + .../licenses/jmespath-java-1.11.390.jar.sha1 | 1 + .../repositories/s3/S3Service.java | 27 ++++++++++--------- .../s3/RepositoryCredentialsTests.java | 7 +++-- 10 files changed, 27 insertions(+), 21 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.390.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.390.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.390.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.390.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 7f0ca209db797..8811e0554c04f 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -32,19 +32,23 @@ esplugin { } versions << [ - 'aws': '1.11.223' + 'aws': '1.11.390' ] dependencies { compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}" compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}" compile "com.amazonaws:aws-java-sdk-core:${versions.aws}" + compile "com.amazonaws:jmespath-java:${versions.aws}" compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" compile "commons-logging:commons-logging:${versions.commonslogging}" compile "commons-codec:commons-codec:${versions.commonscodec}" + compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" compile 'com.fasterxml.jackson.core:jackson-databind:2.6.7.1' compile 'com.fasterxml.jackson.core:jackson-annotations:2.6.0' + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + compile 'joda-time:joda-time:2.10' // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here, // and whitelist this hack in JarHell @@ -53,6 +57,7 @@ dependencies { dependencyLicenses { mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk' + mapping from: /jmespath-java.*/, to: 'aws-java-sdk' mapping from: /jackson-.*/, to: 'jackson' mapping from: /jaxb-.*/, to: 'jaxb' } diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 deleted file mode 100644 index 9890dd8d600b3..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.223.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c3993cb44f5856fa721b7b7ccfc266377c0bf9c0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.390.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.390.jar.sha1 new file mode 100644 index 0000000000000..8213b5ed6937c --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.390.jar.sha1 @@ -0,0 +1 @@ +80ac46e2071c83f6f4ba12b558218ea0364801aa \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 deleted file mode 100644 index d5bc9d30308dc..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.223.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c24e6ebe108c60a08098aeaad5ae0b6a5a77b618 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.390.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.390.jar.sha1 new file mode 100644 index 0000000000000..4f16cb7761986 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.390.jar.sha1 @@ -0,0 +1 @@ +b9a6cd1620032b26d304fabcaa6fc1513762c670 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 deleted file mode 100644 index fe12b2d4847fa..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.223.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c2ef96732e22d97952fbcd0a94f1dc376d157eda \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.390.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.390.jar.sha1 new file mode 100644 index 0000000000000..2e9e862da64b6 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.390.jar.sha1 @@ -0,0 +1 @@ +28aeeb721137787ff5ead5196fe6be022b82b5d6 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.390.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.390.jar.sha1 new file mode 100644 index 0000000000000..f765ae554493f --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.390.jar.sha1 @@ -0,0 +1 @@ +8bb45018488e51df14bd8f507c47148bab777507 \ No newline at end of file diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index b177686bd71a6..ac5baaa8f6df4 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -23,11 +23,14 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; +import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.internal.Constants; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.AbstractComponent; @@ -93,19 +96,17 @@ public AmazonS3Reference client(String clientName) { } } - private AmazonS3 buildClient(S3ClientSettings clientSettings) { - final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings); - final ClientConfiguration configuration = buildConfiguration(clientSettings); - final AmazonS3 client = buildClient(credentials, configuration); - if (Strings.hasText(clientSettings.endpoint)) { - client.setEndpoint(clientSettings.endpoint); - } - return client; - } - // proxy for testing - AmazonS3 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) { - return new AmazonS3Client(credentials, configuration); + AmazonS3 buildClient(final S3ClientSettings clientSettings) { + final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); + builder.withCredentials(buildCredentials(logger, clientSettings)); + builder.withClientConfiguration(buildConfiguration(clientSettings)); + + final String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME; + logger.debug(() -> new ParameterizedMessage("using endpoint [{}]", endpoint)); + builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)); + + return builder.build(); } // pkg private for tests diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java index 7eb603b4b78e5..17797a5758312 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.repositories.s3; -import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; @@ -70,9 +69,9 @@ static final class ProxyS3Service extends S3Service { } @Override - AmazonS3 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) { - final AmazonS3 client = super.buildClient(credentials, configuration); - return new ClientAndCredentials(client, credentials); + AmazonS3 buildClient(final S3ClientSettings clientSettings) { + final AmazonS3 client = super.buildClient(clientSettings); + return new ClientAndCredentials(client, buildCredentials(logger, clientSettings)); } } From 0a10e2c58e7a28233da8021fdf97c15174e8e252 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 12 Sep 2018 12:28:21 +0200 Subject: [PATCH 9/9] Apply feedback + update to 1.11.406 --- plugins/repository-s3/build.gradle | 2 +- .../licenses/aws-java-sdk-core-1.11.390.jar.sha1 | 1 - .../licenses/aws-java-sdk-core-1.11.406.jar.sha1 | 1 + .../licenses/aws-java-sdk-kms-1.11.390.jar.sha1 | 1 - .../licenses/aws-java-sdk-kms-1.11.406.jar.sha1 | 1 + .../licenses/aws-java-sdk-s3-1.11.390.jar.sha1 | 1 - .../licenses/aws-java-sdk-s3-1.11.406.jar.sha1 | 1 + .../licenses/jmespath-java-1.11.390.jar.sha1 | 1 - .../licenses/jmespath-java-1.11.406.jar.sha1 | 1 + .../org/elasticsearch/repositories/s3/S3Service.java | 12 ++++++++++-- 10 files changed, 15 insertions(+), 7 deletions(-) delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.390.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.390.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.390.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.390.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1 diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 633a634105c48..c56a9a8259af3 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -32,7 +32,7 @@ esplugin { } versions << [ - 'aws': '1.11.390' + 'aws': '1.11.406' ] dependencies { diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.390.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.390.jar.sha1 deleted file mode 100644 index 8213b5ed6937c..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.390.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -80ac46e2071c83f6f4ba12b558218ea0364801aa \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1 new file mode 100644 index 0000000000000..415373b275ead --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.406.jar.sha1 @@ -0,0 +1 @@ +43f3b7332d4d527bbf34d4ac6be094f3dabec6de \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.390.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.390.jar.sha1 deleted file mode 100644 index 4f16cb7761986..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.390.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b9a6cd1620032b26d304fabcaa6fc1513762c670 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1 new file mode 100644 index 0000000000000..f0eb9b7175238 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.11.406.jar.sha1 @@ -0,0 +1 @@ +e29854e58dc20f5453c1da7e580a5921b1e9714a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.390.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.390.jar.sha1 deleted file mode 100644 index 2e9e862da64b6..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.390.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -28aeeb721137787ff5ead5196fe6be022b82b5d6 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1 new file mode 100644 index 0000000000000..e57fd11c82980 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.406.jar.sha1 @@ -0,0 +1 @@ +5c3c2c57b076602b3aeef841c63e5848ec52b00d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.390.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.390.jar.sha1 deleted file mode 100644 index f765ae554493f..0000000000000 --- a/plugins/repository-s3/licenses/jmespath-java-1.11.390.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8bb45018488e51df14bd8f507c47148bab777507 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1 new file mode 100644 index 0000000000000..bbb9b562a2fd9 --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.406.jar.sha1 @@ -0,0 +1 @@ +06c291d1029943d4968a36fadffa3b71a6d8b4e4 \ No newline at end of file diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index ac5baaa8f6df4..a431f4da1fdf8 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -30,7 +30,6 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.internal.Constants; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.component.AbstractComponent; @@ -103,7 +102,16 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) { builder.withClientConfiguration(buildConfiguration(clientSettings)); final String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME; - logger.debug(() -> new ParameterizedMessage("using endpoint [{}]", endpoint)); + logger.debug("using endpoint [{}]", endpoint); + + // If the endpoint configuration isn't set on the builder then the default behaviour is to try + // and work out what region we are in and use an appropriate endpoint - see AwsClientBuilder#setRegion. + // In contrast, directly-constructed clients use s3.amazonaws.com unless otherwise instructed. We currently + // use a directly-constructed client, and need to keep the existing behaviour to avoid a breaking change, + // so to move to using the builder we must set it explicitly to keep the existing behaviour. + // + // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too) + // so this change removes that usage of a deprecated API. builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)); return builder.build();