From 26ff16ea7be8ea590668e872d1d1b587af92dea6 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Mon, 8 Jul 2019 14:55:00 +0800 Subject: [PATCH 1/7] add disable_chunked_encoding configuration --- docs/plugins/repository-s3.asciidoc | 7 +++++ .../repositories/s3/S3ClientSettings.java | 26 ++++++++++++++----- .../repositories/s3/S3Service.java | 4 +++ .../s3/S3ClientSettingsTests.java | 7 +++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 48b03f1abc156..0513deba88048 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -153,6 +153,13 @@ settings belong in the `elasticsearch.yml` file. https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS documentation] for details). Defaults to `false`. +`disable_chunked_encoding`:: + Whether chunked encoding should be disabled or not. If `true`, the + chunked encoding will be disabled. If `false`, the chunked encoding will + be automatically determined by the AWS Java SDK (See + https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#withChunkedEncodingDisabled-java.lang.Boolean-[AWS + documentation] for details). Defaults to `false`. + [[repository-s3-path-style-deprecation]] NOTE: In versions `7.0`, `7.1`, `7.2` and `7.3` all bucket operations used the https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[now-deprecated] diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index ae2bd2e905bf6..fee00786a2ab3 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -99,6 +99,10 @@ final class S3ClientSettings { static final Setting.AffixSetting USE_PATH_STYLE_ACCESS = Setting.affixKeySetting(PREFIX, "path_style_access", key -> Setting.boolSetting(key, false, Property.NodeScope)); + /** Whether chunked encoding should be disabled or not (Default is false). */ + static final Setting.AffixSetting DISABLE_CHUNKED_ENCODING = Setting.affixKeySetting(PREFIX, "disable_chunked_encoding", + key -> Setting.boolSetting(key, false, Property.NodeScope)); + /** Credentials to authenticate with s3. */ final S3BasicCredentials credentials; @@ -134,10 +138,13 @@ final class S3ClientSettings { /** Whether the s3 client should use path style access. */ final boolean pathStyleAccess; + /** Whether chunked encoding should be disabled or not. */ + final boolean disableChunkedEncoding; + private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis, int maxRetries, boolean throttleRetries, - boolean pathStyleAccess) { + boolean pathStyleAccess, boolean disableChunkedEncoding) { this.credentials = credentials; this.endpoint = endpoint; this.protocol = protocol; @@ -149,6 +156,7 @@ private S3ClientSettings(S3BasicCredentials credentials, String endpoint, Protoc this.maxRetries = maxRetries; this.throttleRetries = throttleRetries; this.pathStyleAccess = pathStyleAccess; + this.disableChunkedEncoding = disableChunkedEncoding; } /** @@ -172,6 +180,8 @@ S3ClientSettings refine(RepositoryMetaData metadata) { final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); final boolean usePathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); + final boolean newDisableChunkedEncoding = getRepoSettingOrDefault( + DISABLE_CHUNKED_ENCODING, normalizedSettings, disableChunkedEncoding); final S3BasicCredentials newCredentials; if (checkDeprecatedCredentials(repoSettings)) { newCredentials = loadDeprecatedCredentials(repoSettings); @@ -180,7 +190,8 @@ S3ClientSettings refine(RepositoryMetaData metadata) { } if (Objects.equals(endpoint, newEndpoint) && protocol == newProtocol && Objects.equals(proxyHost, newProxyHost) && proxyPort == newProxyPort && newReadTimeoutMillis == readTimeoutMillis && maxRetries == newMaxRetries - && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials)) { + && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials) + && newDisableChunkedEncoding == disableChunkedEncoding) { return this; } return new S3ClientSettings( @@ -194,7 +205,8 @@ S3ClientSettings refine(RepositoryMetaData metadata) { newReadTimeoutMillis, newMaxRetries, newThrottleRetries, - usePathStyleAccess + usePathStyleAccess, + newDisableChunkedEncoding ); } @@ -282,7 +294,8 @@ static S3ClientSettings getClientSettings(final Settings settings, final String Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), getConfigValue(settings, clientName, MAX_RETRIES_SETTING), getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), - getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS) + getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS), + getConfigValue(settings, clientName, DISABLE_CHUNKED_ENCODING) ); } } @@ -305,13 +318,14 @@ public boolean equals(final Object o) { protocol == that.protocol && Objects.equals(proxyHost, that.proxyHost) && Objects.equals(proxyUsername, that.proxyUsername) && - Objects.equals(proxyPassword, that.proxyPassword); + Objects.equals(proxyPassword, that.proxyPassword) && + Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding); } @Override public int hashCode() { return Objects.hash(credentials, endpoint, protocol, proxyHost, proxyPort, proxyUsername, proxyPassword, - readTimeoutMillis, maxRetries, throttleRetries); + readTimeoutMillis, maxRetries, throttleRetries, disableChunkedEncoding); } private static T getConfigValue(Settings settings, String clientName, diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 3b232354ddfea..1fdec450a136f 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -157,6 +157,10 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) { if (clientSettings.pathStyleAccess) { builder.enablePathStyleAccess(); } + + if (clientSettings.disableChunkedEncoding) { + builder.withChunkedEncodingDisabled(true); + } return builder.build(); } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index 312d9649aa375..9f18d1588047f 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -151,4 +151,11 @@ public void testPathStyleAccessCanBeSet() { assertThat(settings.get("default").pathStyleAccess, is(false)); assertThat(settings.get("other").pathStyleAccess, is(true)); } + + public void testUseChunkedEncodingCanBeSet() { + final Map settings = S3ClientSettings.load( + Settings.builder().put("s3.client.other.disable_chunked_encoding", true).build()); + assertThat(settings.get("default").disableChunkedEncoding, is(false)); + assertThat(settings.get("other").disableChunkedEncoding, is(true)); + } } From 08aa85070deb76aa43e666161a7282c4da18488a Mon Sep 17 00:00:00 2001 From: wujinhu Date: Mon, 8 Jul 2019 20:32:31 +0800 Subject: [PATCH 2/7] optimze code & doc --- docs/plugins/repository-s3.asciidoc | 12 ++++++------ .../org/elasticsearch/repositories/s3/S3Service.java | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 0513deba88048..879493b7fd6e6 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -153,6 +153,12 @@ settings belong in the `elasticsearch.yml` file. https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setPathStyleAccessEnabled-java.lang.Boolean-[AWS documentation] for details). Defaults to `false`. +[[repository-s3-path-style-deprecation]] +NOTE: In versions `7.0`, `7.1`, `7.2` and `7.3` all bucket operations used the +https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[now-deprecated] +path style access pattern. If your deployment requires the path style access +pattern then you should set this setting to `true` when upgrading. + `disable_chunked_encoding`:: Whether chunked encoding should be disabled or not. If `true`, the chunked encoding will be disabled. If `false`, the chunked encoding will @@ -160,12 +166,6 @@ settings belong in the `elasticsearch.yml` file. https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#withChunkedEncodingDisabled-java.lang.Boolean-[AWS documentation] for details). Defaults to `false`. -[[repository-s3-path-style-deprecation]] -NOTE: In versions `7.0`, `7.1`, `7.2` and `7.3` all bucket operations used the -https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/[now-deprecated] -path style access pattern. If your deployment requires the path style access -pattern then you should set this setting to `true` when upgrading. - [float] [[repository-s3-compatible-services]] ===== S3-compatible services diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 1fdec450a136f..f7ae303a1c2aa 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -157,9 +157,8 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) { if (clientSettings.pathStyleAccess) { builder.enablePathStyleAccess(); } - if (clientSettings.disableChunkedEncoding) { - builder.withChunkedEncodingDisabled(true); + builder.disableChunkedEncoding(); } return builder.build(); } From e82f4f314e427fcf8124424cc232789f12ec0c25 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Thu, 11 Jul 2019 16:18:49 +0800 Subject: [PATCH 3/7] integration tests support randomly set disable_chunked_encoding --- docs/plugins/repository-s3.asciidoc | 14 ++++++++----- plugins/repository-s3/build.gradle | 5 ++++- .../repositories/s3/AmazonS3Fixture.java | 21 ++++++++++++------- .../20_repository_permanent_credentials.yml | 1 + .../30_repository_temporary_credentials.yml | 1 + .../40_repository_ec2_credentials.yml | 1 + .../50_repository_ecs_credentials.yml | 1 + 7 files changed, 31 insertions(+), 13 deletions(-) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 879493b7fd6e6..c5bd8dbe474c2 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -160,11 +160,15 @@ path style access pattern. If your deployment requires the path style access pattern then you should set this setting to `true` when upgrading. `disable_chunked_encoding`:: - Whether chunked encoding should be disabled or not. If `true`, the - chunked encoding will be disabled. If `false`, the chunked encoding will - be automatically determined by the AWS Java SDK (See - https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#withChunkedEncodingDisabled-java.lang.Boolean-[AWS - documentation] for details). Defaults to `false`. + + Whether chunked encoding should be disabled or not. If `false`, chunked + encoding is enabled and will be used where appropriate. If `true`, chunked + encoding is disabled and will not be used, which may mean that snapshot + operations consume more resources and take longer to complete. It should + only be set to `true` if you are using a storage service that does not + support chunked encoding. See the + https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#disableChunkedEncoding--[AWS + Java SDK documentation] for details. Defaults to `false`. [float] [[repository-s3-compatible-services]] diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 074ac3635c6bb..505d531ce67b8 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -101,6 +101,8 @@ String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs") String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs") +boolean s3DisableChunkedEncoding = (new Random()).nextBoolean() + // If all these variables are missing then we are testing against the internal fixture instead, which has the following // credentials hard-coded in. @@ -257,7 +259,8 @@ processTestResources { 'ec2_bucket': s3EC2Bucket, 'ec2_base_path': s3EC2BasePath, 'ecs_bucket': s3ECSBucket, - 'ecs_base_path': s3ECSBasePath + 'ecs_base_path': s3ECSBasePath, + 'disable_chunked_encoding': s3DisableChunkedEncoding, ] inputs.properties(expansions) MavenFilteringHack.filter(it, expansions) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 51b1d5159edfe..b019ac75a3aab 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.test.fixture.AbstractHttpFixture; import com.amazonaws.util.DateUtils; +import com.amazonaws.util.IOUtils; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; @@ -216,13 +217,13 @@ private PathTrie defaultHandlers(final Map bucke final String destObjectName = objectName(request.getParameters()); - // This is a chunked upload request. We should have the header "Content-Encoding : aws-chunked,gzip" - // to detect it but it seems that the AWS SDK does not follow the S3 guidelines here. - // - // See https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html - // String headerDecodedContentLength = request.getHeader("X-amz-decoded-content-length"); if (headerDecodedContentLength != null) { + // This is a chunked upload request. We should have the header "Content-Encoding : aws-chunked,gzip" + // to detect it but it seems that the AWS SDK does not follow the S3 guidelines here. + // + // See https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html + // int contentLength = Integer.valueOf(headerDecodedContentLength); // Chunked requests have a payload like this: @@ -246,9 +247,15 @@ private PathTrie defaultHandlers(final Map bucke destBucket.objects.put(destObjectName, bytes); return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); } - } + } else { + // Read from body directly + try (BufferedInputStream inputStream = new BufferedInputStream(new ByteArrayInputStream(request.getBody()))) { + byte[] bytes = IOUtils.toByteArray(inputStream); - return newInternalError(request.getId(), "Something is wrong with this PUT request"); + destBucket.objects.put(destObjectName, bytes); + return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); + } + } }) ); diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml index d319bf8984a97..57b2e42bb2503 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml @@ -15,6 +15,7 @@ setup: base_path: "${permanent_base_path}" canned_acl: private storage_class: standard + disable_chunked_encoding: ${disable_chunked_encoding} # Remove the snapshots, if a previous test failed to delete them. This is # useful for third party tests that runs the test against a real external service. diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml index 3ad6c3959634b..1dff4d1ae74f1 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml @@ -15,6 +15,7 @@ setup: base_path: "${temporary_base_path}" canned_acl: private storage_class: standard + disable_chunked_encoding: ${disable_chunked_encoding} --- "Snapshot and Restore with repository-s3 using temporary credentials": diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml index fa1d3fc10fb13..b7c22eccf092e 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml @@ -15,6 +15,7 @@ setup: base_path: "${ec2_base_path}" canned_acl: private storage_class: standard + disable_chunked_encoding: ${disable_chunked_encoding} --- "Snapshot and Restore with repository-s3 using ec2 credentials": diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml index 99736fb25ff24..ba839d3fb98f2 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml @@ -15,6 +15,7 @@ setup: base_path: "${ecs_base_path}" canned_acl: private storage_class: standard + disable_chunked_encoding: ${disable_chunked_encoding} --- "Snapshot and Restore with repository-s3 using ecs credentials": From a6bf9859aea29fb8261e0fbeba112ac4fa2f4bb7 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Thu, 11 Jul 2019 17:36:36 +0800 Subject: [PATCH 4/7] add extra checks for uploading objects --- plugins/repository-s3/build.gradle | 3 ++- .../elasticsearch/repositories/s3/AmazonS3Fixture.java | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 505d531ce67b8..60dd64b4c971b 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -231,7 +231,8 @@ task s3FixtureProperties { "s3Fixture.temporary_key" : s3TemporaryAccessKey, "s3Fixture.temporary_session_token": s3TemporarySessionToken, "s3Fixture.ec2_bucket_name" : s3EC2Bucket, - "s3Fixture.ecs_bucket_name" : s3ECSBucket + "s3Fixture.ecs_bucket_name" : s3ECSBucket, + "s3Fixture.disableChunkedEncoding" : s3DisableChunkedEncoding ] doLast { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index b019ac75a3aab..2e71c148ad839 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -76,6 +76,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture { /** Request handlers for the requests made by the S3 client **/ private final PathTrie handlers; + private final boolean disableChunkedEncoding; /** * Creates a {@link AmazonS3Fixture} */ @@ -93,6 +94,8 @@ private AmazonS3Fixture(final String workingDir, Properties properties) { randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10)); this.handlers = defaultHandlers(buckets, ec2Bucket, ecsBucket); + + this.disableChunkedEncoding = Boolean.parseBoolean(prop(properties, "s3Fixture.disableChunkedEncoding")); } private static String nonAuthPath(Request request) { @@ -219,6 +222,9 @@ private PathTrie defaultHandlers(final Map bucke String headerDecodedContentLength = request.getHeader("X-amz-decoded-content-length"); if (headerDecodedContentLength != null) { + if (disableChunkedEncoding) { + return newInternalError(request.getId(), "Something is wrong with this PUT request"); + } // This is a chunked upload request. We should have the header "Content-Encoding : aws-chunked,gzip" // to detect it but it seems that the AWS SDK does not follow the S3 guidelines here. // @@ -248,6 +254,9 @@ private PathTrie defaultHandlers(final Map bucke return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); } } else { + if (!disableChunkedEncoding) { + return newInternalError(request.getId(), "Something is wrong with this PUT request"); + } // Read from body directly try (BufferedInputStream inputStream = new BufferedInputStream(new ByteArrayInputStream(request.getBody()))) { byte[] bytes = IOUtils.toByteArray(inputStream); From feb8ad191c5c1c6f6071e7d105438787412cd545 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Fri, 12 Jul 2019 13:44:48 +0800 Subject: [PATCH 5/7] code optimizations --- plugins/repository-s3/build.gradle | 6 ++---- .../org/elasticsearch/repositories/s3/AmazonS3Fixture.java | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 60dd64b4c971b..5bffc82a6562f 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -101,7 +101,7 @@ String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs") String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs") -boolean s3DisableChunkedEncoding = (new Random()).nextBoolean() +boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(project.rootProject.testSeed.tokenize(':').get(0), 16))).nextBoolean() // If all these variables are missing then we are testing against the internal fixture instead, which has the following // credentials hard-coded in. @@ -235,9 +235,7 @@ task s3FixtureProperties { "s3Fixture.disableChunkedEncoding" : s3DisableChunkedEncoding ] - doLast { - file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n") - } + file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n") } /** A task to start the AmazonS3Fixture which emulates an S3 service **/ diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index 2e71c148ad839..e0434d1e50f1d 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -254,7 +254,7 @@ private PathTrie defaultHandlers(final Map bucke return new Response(RestStatus.OK.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); } } else { - if (!disableChunkedEncoding) { + if (disableChunkedEncoding == false) { return newInternalError(request.getId(), "Something is wrong with this PUT request"); } // Read from body directly From 75581957ae6bb3ba53ba5228b1ce311638ed2e92 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Mon, 15 Jul 2019 12:53:46 +0800 Subject: [PATCH 6/7] fix build issue & force to updte s3Fixture.properties --- plugins/repository-s3/build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 5bffc82a6562f..1175d47b6f624 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -235,7 +235,10 @@ task s3FixtureProperties { "s3Fixture.disableChunkedEncoding" : s3DisableChunkedEncoding ] - file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n") + outputs.upToDateWhen { false } + doLast { + file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n") + } } /** A task to start the AmazonS3Fixture which emulates an S3 service **/ From dec263e09e10290de6c6b4e8e659f1a3599acff3 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Thu, 18 Jul 2019 15:57:39 +0800 Subject: [PATCH 7/7] revert outputs change in build.gradle --- plugins/repository-s3/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 1175d47b6f624..af10aa079ee7c 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -235,7 +235,6 @@ task s3FixtureProperties { "s3Fixture.disableChunkedEncoding" : s3DisableChunkedEncoding ] - outputs.upToDateWhen { false } doLast { file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n") }