diff --git a/LICENSE-binary b/LICENSE-binary
index 851d839cdbd8d..0c8199da1be15 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -363,8 +363,7 @@ org.objenesis:objenesis:2.6
org.xerial.snappy:snappy-java:1.1.10.1
org.yaml:snakeyaml:2.0
org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-software.amazon.awssdk:bundle:jar:2.19.12
-software.amazon.awssdk.crt:aws-crt:0.21.0
+software.amazon.awssdk:bundle:jar:2.20.128
--------------------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 809084cb3f7da..47fa59cc67161 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -184,9 +184,8 @@
900
1.12.367
2.7.1
- 2.19.12
- 1.0.1
- 0.21.0
+ 2.20.128
+ 1.0.1
1.11.2
2.1
0.7
@@ -1154,12 +1153,7 @@
software.amazon.eventstream
eventstream
- ${aws.evenstream.version}
-
-
- software.amazon.awssdk.crt
- aws-crt
- ${awscrt.version}
+ ${aws.eventstream.version}
org.apache.mina
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 3591ab4ea5a50..66154e2840e34 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -518,10 +518,6 @@
bundle
compile
-
- software.amazon.awssdk.crt
- aws-crt
-
software.amazon.eventstream
eventstream
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index 1b2c129a6428a..98c72d276628e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -37,6 +37,7 @@
import software.amazon.awssdk.services.s3.S3BaseClientBuilder;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration;
import software.amazon.awssdk.transfer.s3.S3TransferManager;
import org.apache.commons.lang3.StringUtils;
@@ -98,17 +99,25 @@ public S3AsyncClient createS3AsyncClient(
Configuration conf = getConf();
String bucket = uri.getHost();
+
NettyNioAsyncHttpClient.Builder httpClientBuilder = AWSClientConfig
.createAsyncHttpClientBuilder(conf)
.proxyConfiguration(AWSClientConfig.createAsyncProxyConfiguration(conf, bucket));
+
+ MultipartConfiguration multipartConfiguration = MultipartConfiguration.builder()
+ .minimumPartSizeInBytes(parameters.getMinimumPartSize())
+ .thresholdInBytes(parameters.getMultiPartThreshold())
+ .build();
+
return configureClientBuilder(S3AsyncClient.builder(), parameters, conf, bucket)
.httpClientBuilder(httpClientBuilder)
+ .multipartConfiguration(multipartConfiguration)
+ .multipartEnabled(true)
.build();
}
@Override
public S3TransferManager createS3TransferManager(final S3AsyncClient s3AsyncClient) {
-
return S3TransferManager.builder()
.s3Client(s3AsyncClient)
.build();
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 305695f2ef5ac..43f0b9c98cdd6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -982,6 +982,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
.withRequesterPays(conf.getBoolean(ALLOW_REQUESTER_PAYS, DEFAULT_ALLOW_REQUESTER_PAYS))
.withExecutionInterceptors(auditManager.createExecutionInterceptors())
.withMinimumPartSize(partSize)
+ .withMultipartThreshold(multiPartThreshold)
.withTransferManagerExecutor(unboundedThreadPool)
.withRegion(region);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
index fa2c0769d26a3..d4504cd08d74c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -151,6 +151,11 @@ final class S3ClientCreationParameters {
*/
private long minimumPartSize;
+ /**
+ * Threshold for multipart operations.
+ */
+ private long multiPartThreshold;
+
/**
* Executor that the transfer manager will use to execute background tasks.
*/
@@ -337,6 +342,25 @@ public S3ClientCreationParameters withMinimumPartSize(
return this;
}
+ /**
+ * Get the threshold for multipart operations.
+ * @return multipart threshold
+ */
+ public long getMultiPartThreshold() {
+ return multiPartThreshold;
+ }
+
+ /**
+ * Set the threshold for multipart operations.
+ * @param value new value
+ * @return the builder
+ */
+ public S3ClientCreationParameters withMultipartThreshold(
+ final long value) {
+ multiPartThreshold = value;
+ return this;
+ }
+
/**
* Get the executor that the transfer manager will use to execute background tasks.
* @return part size