From 586cff1b1c6006762e4b504e215dc0acc88c309c Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Mon, 29 Jul 2019 12:59:57 -0700 Subject: [PATCH 1/6] HADOOP-16445. Allow separate custom signing algorithms for S3 and DDB --- .../org/apache/hadoop/fs/s3a/Constants.java | 30 +++- .../hadoop/fs/s3a/DefaultS3ClientFactory.java | 3 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 2 + .../org/apache/hadoop/fs/s3a/S3AUtils.java | 75 +++++++++- .../fs/s3a/s3guard/DynamoDBClientFactory.java | 3 +- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 138 ++++++++++++++++++ 6 files changed, 245 insertions(+), 6 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 5c45ce6e58c4f..d8e3b0ed61d53 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -343,9 +343,37 @@ private Constants() { public static final String SERVER_SIDE_ENCRYPTION_KEY = "fs.s3a.server-side-encryption.key"; - //override signature algorithm used for signing requests + /** + * List of custom Signers. The signer class will be loaded, and the signer + * name will be associated with this signer class in the S3 SDK. e.g. Single + * CustomSigner -> 'CustomSigner:org.apache...CustomSignerClass Multiple + * CustomSigners -> 'CSigner1:CustomSignerClass1,CSigner2:CustomerSignerClass2 + */ + public static final String CUSTOM_SIGNERS = "fs.s3a.custom.signers"; + + /** + * There's 3 parameters that can be used to specify a non-default signing + * algorithm. fs.s3a.signing-algorithm - This property has existed for the + * longest time. If specified, without either of the other 2 properties being + * specified, this signing algorithm will be used for S3 and DDB (S3Guard). + * The other 2 properties override this value for S3 or DDB. + * fs.s3a.s3.signing-algorithm - Allows overriding the S3 Signing algorithm. + * This does not affect DDB. Specifying this property without specifying + * fs.s3a.signing-algorithm will only update the signing algorithm for S3 + * requests, and the default will be used for DDB fs.s3a.ddb.signing-algorithm + * - Allows overriding the DDB Signing algorithm. This does not affect S3. + * Specifying this property without specifying fs.s3a.signing-algorithm will + * only update the signing algorithm for DDB requests, and the default will be + * used for S3 + */ public static final String SIGNING_ALGORITHM = "fs.s3a.signing-algorithm"; + public static final String SIGNING_ALGORITHM_S3 = + "fs.s3a.s3.signing-algorithm"; + + public static final String SIGNING_ALGORITHM_DDB = + "fs.s3a.ddb.signing-algorithm"; + public static final String S3N_FOLDER_SUFFIX = "_$folder$"; public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size"; public static final String FS_S3A = "s3a"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index 3e9368d10f624..b8942d0f6831e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -55,7 +55,8 @@ public AmazonS3 createS3Client(URI name, final AWSCredentialsProvider credentials, final String userAgentSuffix) throws IOException { Configuration conf = getConf(); - final ClientConfiguration awsConf = S3AUtils.createAwsConf(getConf(), bucket); + final ClientConfiguration awsConf = S3AUtils + .createAwsConfForS3(getConf(), bucket); if (!StringUtils.isEmpty(userAgentSuffix)) { awsConf.setUserAgentSuffix(userAgentSuffix); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 6bdbba30480a4..7880e835cd095 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -300,6 +300,8 @@ public void initialize(URI name, Configuration originalConf) LOG.debug("Initializing S3AFileSystem for {}", bucket); // clone the configuration into one with propagated bucket options Configuration conf = propagateBucketOptions(originalConf, bucket); + // Initialize any custom signers + initCustomSigners(conf); // patch the Hadoop security providers patchSecurityCredentialProviders(conf); // look for delegation token support early. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index a69815d442440..daaa40838cedc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -26,6 +26,8 @@ import com.amazonaws.SdkBaseException; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; +import com.amazonaws.auth.Signer; +import com.amazonaws.auth.SignerFactory; import com.amazonaws.retry.RetryUtils; import com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException; import com.amazonaws.services.dynamodbv2.model.LimitExceededException; @@ -1195,9 +1197,10 @@ public static void deleteWithWarning(FileSystem fs, } /** - * Create a new AWS {@code ClientConfiguration}. - * All clients to AWS services MUST use this for consistent setup - * of connectivity, UA, proxy settings. + * Create a new AWS {@code ClientConfiguration}. All clients to AWS services + * MUST use this or the equivalents for the specific service for + * consistent setup of connectivity, UA, proxy settings. + * * @param conf The Hadoop configuration * @param bucket Optional bucket to use to look up per-bucket proxy secrets * @return new AWS client configuration @@ -1213,6 +1216,38 @@ public static ClientConfiguration createAwsConf(Configuration conf, return awsConf; } + /** + * Create a new AWS {#link ClientConfiguration} S3 clients MUST use + * this for consistent setup of connectivity, UA, proxy settings. + */ + public static ClientConfiguration createAwsConfForS3(Configuration conf, + String bucket) + throws IOException { + ClientConfiguration awsConf = createAwsConf(conf, bucket); + String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM_S3, ""); + if (!signerOverride.isEmpty()) { + LOG.debug("Signer override for S3 = {}", signerOverride); + awsConf.setSignerOverride(signerOverride); + } + return awsConf; + } + + /** + * Create a new AWS {#link ClientConfiguration} DynamoDB clients MUST + * use this for consistent setup of connectivity, UA, proxy settings. + */ + public static ClientConfiguration createAwsConfForDdb(Configuration conf, + String bucket) + throws IOException { + ClientConfiguration awsConf = createAwsConf(conf, bucket); + String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM_DDB, ""); + if (!signerOverride.isEmpty()) { + LOG.debug("Signer override for DDB = {}", signerOverride); + awsConf.setSignerOverride(signerOverride); + } + return awsConf; + } + /** * Initializes all AWS SDK settings related to connection management. * @@ -1303,6 +1338,40 @@ public static void initProxySupport(Configuration conf, } } + /** + * Initialize custom signers and register them with the AWS SDK. + * + * @param conf Hadoop configuration + */ + public static void initCustomSigners(Configuration conf) { + String[] customSigners = conf.getTrimmedStrings(CUSTOM_SIGNERS); + if (customSigners == null || customSigners.length == 0) { + // No custom signers specified, nothing to do. + LOG.debug("No custom signers specified"); + return; + } + + for (String customSigner : customSigners) { + String[] parts = customSigner.split(":"); + if (parts.length != 2) { + String message = + "Invalid format (name:class) for CustomSigner: [" + customSigner + + "]"; + LOG.error(message); + throw new IllegalArgumentException(message); + } + Class clazz = null; + try { + clazz = (Class) conf.getClassByName(parts[1]); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + LOG.debug("Registering Custom Signer - [{}->{}]", parts[0], + clazz.getName()); + SignerFactory.registerSigner(parts[0], clazz); + } + } + /** * Initializes the User-Agent header to send in HTTP requests to AWS * services. We always include the Hadoop version number. The user also diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java index 9e1d2f41b5196..7ceaf7a2d934a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java @@ -80,7 +80,8 @@ public AmazonDynamoDB createDynamoDBClient(String defaultRegion, "Should have been configured before usage"); final Configuration conf = getConf(); - final ClientConfiguration awsConf = S3AUtils.createAwsConf(conf, bucket); + final ClientConfiguration awsConf = S3AUtils + .createAwsConfForDdb(conf, bucket); final String region = getRegion(conf, defaultRegion); LOG.debug("Creating DynamoDB client in region {}", region); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 959c424023120..537599dc297d2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -19,17 +19,24 @@ package org.apache.hadoop.fs.s3a; import com.amazonaws.ClientConfiguration; +import com.amazonaws.SignableRequest; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.Signer; +import com.amazonaws.auth.SignerFactory; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.S3ClientOptions; +import java.io.IOException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3native.S3xLoginHelper; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -617,4 +624,135 @@ public void testSecurityCredentialPropagationEndToEnd() throws Exception { "override,base"); } + @Test(timeout = 10_000L) + public void testS3SpecificSignerOverride() throws IOException { + ClientConfiguration clientConfiguration = null; + Configuration config; + + String signerOverride = "testSigner"; + String s3SignerOverride = "testS3Signer"; + + // Default SIGNING_ALGORITHM, overridden for S3 only + config = new Configuration(); + config.set(SIGNING_ALGORITHM_S3, s3SignerOverride); + clientConfiguration = S3AUtils.createAwsConfForS3(config, "dontcare"); + Assert.assertEquals(s3SignerOverride, + clientConfiguration.getSignerOverride()); + clientConfiguration = S3AUtils.createAwsConfForDdb(config, "dontcare"); + Assert.assertNull(clientConfiguration.getSignerOverride()); + + // Configured base SIGNING_ALGORITHM, overridden for S3 only + config = new Configuration(); + config.set(SIGNING_ALGORITHM, signerOverride); + config.set(SIGNING_ALGORITHM_S3, s3SignerOverride); + clientConfiguration = S3AUtils.createAwsConfForS3(config, "dontcare"); + Assert.assertEquals(s3SignerOverride, + clientConfiguration.getSignerOverride()); + clientConfiguration = S3AUtils.createAwsConfForDdb(config, "dontcare"); + Assert + .assertEquals(signerOverride, clientConfiguration.getSignerOverride()); + } + + @Test(timeout = 10_000L) + public void testDdbSpecificSignerOverride() throws IOException { + ClientConfiguration clientConfiguration = null; + Configuration config; + + String signerOverride = "testSigner"; + String ddbSignerOverride = "testDdbSigner"; + + // Default SIGNING_ALGORITHM, overridden for S3 + config = new Configuration(); + config.set(SIGNING_ALGORITHM_DDB, ddbSignerOverride); + clientConfiguration = S3AUtils.createAwsConfForDdb(config, "dontcare"); + Assert.assertEquals(ddbSignerOverride, + clientConfiguration.getSignerOverride()); + clientConfiguration = S3AUtils.createAwsConfForS3(config, "dontcare"); + Assert.assertNull(clientConfiguration.getSignerOverride()); + + // Configured base SIGNING_ALGORITHM, overridden for S3 + config = new Configuration(); + config.set(SIGNING_ALGORITHM, signerOverride); + config.set(SIGNING_ALGORITHM_DDB, ddbSignerOverride); + clientConfiguration = S3AUtils.createAwsConfForDdb(config, "dontcare"); + Assert.assertEquals(ddbSignerOverride, + clientConfiguration.getSignerOverride()); + clientConfiguration = S3AUtils.createAwsConfForS3(config, "dontcare"); + Assert + .assertEquals(signerOverride, clientConfiguration.getSignerOverride()); + } + + // Expecting generic Exception.class to handle future implementation changes. + // For now, this is an NPE + @Test(timeout = 10_000L, expected = Exception.class) + public void testCustomSignerFailureIfNotRegistered() { + Signer s1 = SignerFactory.createSigner("testsigner1", null); + } + + @Test(timeout = 10_000L) + public void testCustomSignerInitialization() { + Configuration config = new Configuration(); + SignerForTest1.reset(); + SignerForTest2.reset(); + config.set(CUSTOM_SIGNERS, "testsigner1:" + SignerForTest1.class.getName()); + initCustomSigners(config); + Signer s1 = SignerFactory.createSigner("testsigner1", null); + s1.sign(null, null); + Assert.assertEquals(true, SignerForTest1.initialized); + } + + @Test(timeout = 10_000L) + public void testMultipleCustomSignerInitialization() { + Configuration config = new Configuration(); + SignerForTest1.reset(); + SignerForTest2.reset(); + config.set(CUSTOM_SIGNERS, + "testsigner1:" + SignerForTest1.class.getName() + "," + "testsigner2:" + + SignerForTest2.class.getName()); + initCustomSigners(config); + Signer s1 = SignerFactory.createSigner("testsigner1", null); + s1.sign(null, null); + Assert.assertEquals(true, SignerForTest1.initialized); + + Signer s2 = SignerFactory.createSigner("testsigner2", null); + s2.sign(null, null); + Assert.assertEquals(true, SignerForTest2.initialized); + } + + + /** + * SignerForTest1. + */ + @Private + public static class SignerForTest1 implements Signer { + + private static boolean initialized = false; + + @Override + public void sign(SignableRequest request, AWSCredentials credentials) { + initialized = true; + } + + public static void reset() { + initialized = false; + } + } + + /** + * SignerForTest2. + */ + @Private + public static class SignerForTest2 implements Signer { + + private static boolean initialized = false; + + @Override + public void sign(SignableRequest request, AWSCredentials credentials) { + initialized = true; + } + + public static void reset() { + initialized = false; + } + } } From 38a3f367cdd895d5308286a792fde4f4639f4e4a Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Wed, 18 Sep 2019 11:51:48 +0530 Subject: [PATCH 2/6] Incorporated review comments, and other changes. --- .../org/apache/hadoop/fs/s3a/Constants.java | 13 +- .../hadoop/fs/s3a/DefaultS3ClientFactory.java | 2 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 8 +- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 111 ++++++---------- .../apache/hadoop/fs/s3a/SignerManager.java | 99 ++++++++++++++ .../hadoop/fs/s3a/auth/STSClientFactory.java | 7 +- .../auth/delegation/SessionTokenBinding.java | 4 +- .../fs/s3a/s3guard/DynamoDBClientFactory.java | 3 +- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 103 +++------------ .../hadoop/fs/s3a/TestSignerManager.java | 124 ++++++++++++++++++ 10 files changed, 310 insertions(+), 164 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index d8e3b0ed61d53..d701cdab38f13 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -369,10 +369,16 @@ private Constants() { public static final String SIGNING_ALGORITHM = "fs.s3a.signing-algorithm"; public static final String SIGNING_ALGORITHM_S3 = - "fs.s3a.s3.signing-algorithm"; + "fs.s3a." + Constants.AWS_SERVICE_IDENTIFIER_S3.toLowerCase() + + ".signing-algorithm"; public static final String SIGNING_ALGORITHM_DDB = - "fs.s3a.ddb.signing-algorithm"; + "fs.s3a." + Constants.AWS_SERVICE_IDENTIFIER_DDB.toLowerCase() + + "signing-algorithm"; + + public static final String SIGNING_ALGORITHM_STS = + "fs.s3a." + Constants.AWS_SERVICE_IDENTIFIER_STS.toLowerCase() + + "signing-algorithm"; public static final String S3N_FOLDER_SUFFIX = "_$folder$"; public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size"; @@ -817,4 +823,7 @@ private Constants() { public static final String S3GUARD_CONSISTENCY_RETRY_INTERVAL_DEFAULT = "2s"; + public static final String AWS_SERVICE_IDENTIFIER_S3 = "S3"; + public static final String AWS_SERVICE_IDENTIFIER_DDB = "DDB"; + public static final String AWS_SERVICE_IDENTIFIER_STS = "STS"; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index b8942d0f6831e..ff8ba1d6d5dac 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -56,7 +56,7 @@ public AmazonS3 createS3Client(URI name, final String userAgentSuffix) throws IOException { Configuration conf = getConf(); final ClientConfiguration awsConf = S3AUtils - .createAwsConfForS3(getConf(), bucket); + .createAwsConf(getConf(), bucket, Constants.AWS_SERVICE_IDENTIFIER_S3); if (!StringUtils.isEmpty(userAgentSuffix)) { awsConf.setUserAgentSuffix(userAgentSuffix); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 7880e835cd095..77e4ef22279a3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -259,6 +259,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private MagicCommitIntegration committerIntegration; private AWSCredentialProviderList credentials; + private SignerManager signerManager; private ITtlTimeProvider ttlTimeProvider; @@ -300,8 +301,6 @@ public void initialize(URI name, Configuration originalConf) LOG.debug("Initializing S3AFileSystem for {}", bucket); // clone the configuration into one with propagated bucket options Configuration conf = propagateBucketOptions(originalConf, bucket); - // Initialize any custom signers - initCustomSigners(conf); // patch the Hadoop security providers patchSecurityCredentialProviders(conf); // look for delegation token support early. @@ -361,6 +360,9 @@ public void initialize(URI name, Configuration originalConf) } useListV1 = (listVersion == 1); + signerManager = new SignerManager(); + signerManager.initCustomSigners(conf);; + // creates the AWS client, including overriding auth chain if // the FS came with a DT // this may do some patching of the configuration (e.g. setting @@ -3055,6 +3057,8 @@ public void close() throws IOException { instrumentation = null; closeAutocloseables(LOG, credentials); cleanupWithLogger(LOG, delegationTokens.orElse(null)); + cleanupWithLogger(LOG, signerManager); + signerManager = null; credentials = null; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index daaa40838cedc..395773a11ddc9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -26,8 +26,6 @@ import com.amazonaws.SdkBaseException; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; -import com.amazonaws.auth.Signer; -import com.amazonaws.auth.SignerFactory; import com.amazonaws.retry.RetryUtils; import com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException; import com.amazonaws.services.dynamodbv2.model.LimitExceededException; @@ -1196,6 +1194,24 @@ public static void deleteWithWarning(FileSystem fs, } } + /** + * Create a new AWS {@code ClientConfiguration}. + * All clients to AWS services MUST use this for consistent setup + * of connectivity, UA, proxy settings. + * @param conf The Hadoop configuration + * @param bucket Optional bucket to use to look up per-bucket proxy secrets + * @return new AWS client configuration + * @throws IOException problem creating AWS client configuration + * + * @deprecated use {@link #createAwsConf(Configuration, String, String)} + */ + @Deprecated + public static ClientConfiguration createAwsConf(Configuration conf, + String bucket) + throws IOException { + return createAwsConf(conf, bucket, null); + } + /** * Create a new AWS {@code ClientConfiguration}. All clients to AWS services * MUST use this or the equivalents for the specific service for @@ -1203,47 +1219,40 @@ public static void deleteWithWarning(FileSystem fs, * * @param conf The Hadoop configuration * @param bucket Optional bucket to use to look up per-bucket proxy secrets + * @param awsServiceIdentifier a string representing the AWS service (S3, + * DDB, etc) for which the ClientConfiguration is being created. * @return new AWS client configuration * @throws IOException problem creating AWS client configuration */ public static ClientConfiguration createAwsConf(Configuration conf, - String bucket) + String bucket, String awsServiceIdentifier) throws IOException { final ClientConfiguration awsConf = new ClientConfiguration(); initConnectionSettings(conf, awsConf); initProxySupport(conf, bucket, awsConf); initUserAgent(conf, awsConf); - return awsConf; - } - - /** - * Create a new AWS {#link ClientConfiguration} S3 clients MUST use - * this for consistent setup of connectivity, UA, proxy settings. - */ - public static ClientConfiguration createAwsConfForS3(Configuration conf, - String bucket) - throws IOException { - ClientConfiguration awsConf = createAwsConf(conf, bucket); - String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM_S3, ""); - if (!signerOverride.isEmpty()) { - LOG.debug("Signer override for S3 = {}", signerOverride); - awsConf.setSignerOverride(signerOverride); - } - return awsConf; - } - - /** - * Create a new AWS {#link ClientConfiguration} DynamoDB clients MUST - * use this for consistent setup of connectivity, UA, proxy settings. - */ - public static ClientConfiguration createAwsConfForDdb(Configuration conf, - String bucket) - throws IOException { - ClientConfiguration awsConf = createAwsConf(conf, bucket); - String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM_DDB, ""); - if (!signerOverride.isEmpty()) { - LOG.debug("Signer override for DDB = {}", signerOverride); - awsConf.setSignerOverride(signerOverride); + if (StringUtils.isNotEmpty(awsServiceIdentifier)) { + String configKey = null; + switch (awsServiceIdentifier) { + case AWS_SERVICE_IDENTIFIER_S3: + configKey = SIGNING_ALGORITHM_S3; + break; + case AWS_SERVICE_IDENTIFIER_DDB: + configKey = SIGNING_ALGORITHM_DDB; + break; + case AWS_SERVICE_IDENTIFIER_STS: + configKey = SIGNING_ALGORITHM_STS; + break; + default: + // Nothing to do. The original signer override is already setup + } + if (configKey != null) { + String signerOverride = conf.getTrimmed(configKey, ""); + if (!signerOverride.isEmpty()) { + LOG.debug("Signer override for {}} = {}", awsServiceIdentifier, signerOverride); + awsConf.setSignerOverride(signerOverride); + } + } } return awsConf; } @@ -1338,40 +1347,6 @@ public static void initProxySupport(Configuration conf, } } - /** - * Initialize custom signers and register them with the AWS SDK. - * - * @param conf Hadoop configuration - */ - public static void initCustomSigners(Configuration conf) { - String[] customSigners = conf.getTrimmedStrings(CUSTOM_SIGNERS); - if (customSigners == null || customSigners.length == 0) { - // No custom signers specified, nothing to do. - LOG.debug("No custom signers specified"); - return; - } - - for (String customSigner : customSigners) { - String[] parts = customSigner.split(":"); - if (parts.length != 2) { - String message = - "Invalid format (name:class) for CustomSigner: [" + customSigner - + "]"; - LOG.error(message); - throw new IllegalArgumentException(message); - } - Class clazz = null; - try { - clazz = (Class) conf.getClassByName(parts[1]); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - LOG.debug("Registering Custom Signer - [{}->{}]", parts[0], - clazz.getName()); - SignerFactory.registerSigner(parts[0], clazz); - } - } - /** * Initializes the User-Agent header to send in HTTP requests to AWS * services. We always include the Hadoop version number. The user also diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java new file mode 100644 index 0000000000000..b96fb18069898 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a; + +import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS; + +import com.amazonaws.auth.Signer; +import com.amazonaws.auth.SignerFactory; +import java.io.Closeable; +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Class to handle custom signers + */ +public class SignerManager implements Closeable { + + private static final Logger LOG = LoggerFactory + .getLogger(SignerManager.class); + + + public SignerManager() { + } + + /** + * Initialize custom signers and register them with the AWS SDK. + * + * @param conf Hadoop configuration + */ + public void initCustomSigners(Configuration conf) { + String[] customSigners = conf.getTrimmedStrings(CUSTOM_SIGNERS); + if (customSigners == null || customSigners.length == 0) { + // No custom signers specified, nothing to do. + LOG.debug("No custom signers specified"); + return; + } + + for (String customSigner : customSigners) { + String[] parts = customSigner.split(":"); + if (parts.length != 2) { + String message = + "Invalid format (Expected name:SignerClass) for CustomSigner: [" + + customSigner + + "]"; + LOG.error(message); + throw new IllegalArgumentException(message); + } + maybeRegisterSigner(parts[0], parts[1], conf); + } + } + + /* + * Make sure the signer class is registered once with the AWS SDK + */ + private static void maybeRegisterSigner(String signerName, + String signerClassName, Configuration conf) { + try { + SignerFactory.getSignerByTypeAndService(signerName, null); + } catch (IllegalArgumentException e) { + // Signer is not registered with the AWS SDK. + // Load the class and register the signer. + Class clazz = null; + try { + clazz = (Class) conf.getClassByName(signerClassName); + } catch (ClassNotFoundException cnfe) { + throw new RuntimeException(String + .format("Signer class [%s] not found for signer [%s]", + signerClassName, signerName), cnfe); + } + LOG.debug("Registering Custom Signer - [{}->{}]", signerName, + clazz.getName()); + synchronized (SignerManager.class) { + SignerFactory.registerSigner(signerName, clazz); + } + } + } + + @Override + public void close() throws IOException { + + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java index 74aca50fa9c53..d19d820bef28c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java @@ -31,6 +31,7 @@ import com.amazonaws.services.securitytoken.model.Credentials; import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest; import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.s3a.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,7 +74,8 @@ public static AWSSecurityTokenServiceClientBuilder builder( final Configuration conf, final String bucket, final AWSCredentialsProvider credentials) throws IOException { - final ClientConfiguration awsConf = S3AUtils.createAwsConf(conf, bucket); + final ClientConfiguration awsConf = S3AUtils.createAwsConf(conf, bucket, + Constants.AWS_SERVICE_IDENTIFIER_STS); String endpoint = conf.getTrimmed(DELEGATION_TOKEN_ENDPOINT, DEFAULT_DELEGATION_TOKEN_ENDPOINT); String region = conf.getTrimmed(DELEGATION_TOKEN_REGION, @@ -99,7 +101,8 @@ public static AWSSecurityTokenServiceClientBuilder builder( final AWSCredentialsProvider credentials, final String stsEndpoint, final String stsRegion) throws IOException { - final ClientConfiguration awsConf = S3AUtils.createAwsConf(conf, bucket); + final ClientConfiguration awsConf = S3AUtils.createAwsConf(conf, bucket, + Constants.AWS_SERVICE_IDENTIFIER_STS); return builder(credentials, awsConf, stsEndpoint, stsRegion); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java index 08d53cf59368e..0f327028d4c0c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java @@ -31,6 +31,7 @@ import com.amazonaws.auth.AWSSessionCredentials; import com.amazonaws.services.securitytoken.AWSSecurityTokenService; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.fs.s3a.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -301,7 +302,8 @@ private synchronized Optional maybeInitSTS() invoker = new Invoker(new S3ARetryPolicy(conf), LOG_EVENT); ClientConfiguration awsConf = - S3AUtils.createAwsConf(conf, uri.getHost()); + S3AUtils.createAwsConf(conf, uri.getHost(), + Constants.AWS_SERVICE_IDENTIFIER_STS); AWSSecurityTokenService tokenService = STSClientFactory.builder(parentAuthChain, awsConf, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java index 7ceaf7a2d934a..5f21250449633 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java @@ -27,6 +27,7 @@ import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.s3a.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,7 +82,7 @@ public AmazonDynamoDB createDynamoDBClient(String defaultRegion, final Configuration conf = getConf(); final ClientConfiguration awsConf = S3AUtils - .createAwsConfForDdb(conf, bucket); + .createAwsConf(conf, bucket, Constants.AWS_SERVICE_IDENTIFIER_DDB); final String region = getRegion(conf, defaultRegion); LOG.debug("Creating DynamoDB client in region {}", region); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 537599dc297d2..614fa53631154 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -19,17 +19,12 @@ package org.apache.hadoop.fs.s3a; import com.amazonaws.ClientConfiguration; -import com.amazonaws.SignableRequest; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.Signer; -import com.amazonaws.auth.SignerFactory; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.S3ClientOptions; import java.io.IOException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -635,20 +630,24 @@ public void testS3SpecificSignerOverride() throws IOException { // Default SIGNING_ALGORITHM, overridden for S3 only config = new Configuration(); config.set(SIGNING_ALGORITHM_S3, s3SignerOverride); - clientConfiguration = S3AUtils.createAwsConfForS3(config, "dontcare"); + clientConfiguration = S3AUtils + .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_S3); Assert.assertEquals(s3SignerOverride, clientConfiguration.getSignerOverride()); - clientConfiguration = S3AUtils.createAwsConfForDdb(config, "dontcare"); + clientConfiguration = S3AUtils + .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_DDB); Assert.assertNull(clientConfiguration.getSignerOverride()); // Configured base SIGNING_ALGORITHM, overridden for S3 only config = new Configuration(); config.set(SIGNING_ALGORITHM, signerOverride); config.set(SIGNING_ALGORITHM_S3, s3SignerOverride); - clientConfiguration = S3AUtils.createAwsConfForS3(config, "dontcare"); + clientConfiguration = S3AUtils + .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_S3); Assert.assertEquals(s3SignerOverride, clientConfiguration.getSignerOverride()); - clientConfiguration = S3AUtils.createAwsConfForDdb(config, "dontcare"); + clientConfiguration = S3AUtils + .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_DDB); Assert .assertEquals(signerOverride, clientConfiguration.getSignerOverride()); } @@ -664,95 +663,25 @@ public void testDdbSpecificSignerOverride() throws IOException { // Default SIGNING_ALGORITHM, overridden for S3 config = new Configuration(); config.set(SIGNING_ALGORITHM_DDB, ddbSignerOverride); - clientConfiguration = S3AUtils.createAwsConfForDdb(config, "dontcare"); + clientConfiguration = S3AUtils + .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_DDB); Assert.assertEquals(ddbSignerOverride, clientConfiguration.getSignerOverride()); - clientConfiguration = S3AUtils.createAwsConfForS3(config, "dontcare"); + clientConfiguration = S3AUtils + .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_S3); Assert.assertNull(clientConfiguration.getSignerOverride()); // Configured base SIGNING_ALGORITHM, overridden for S3 config = new Configuration(); config.set(SIGNING_ALGORITHM, signerOverride); config.set(SIGNING_ALGORITHM_DDB, ddbSignerOverride); - clientConfiguration = S3AUtils.createAwsConfForDdb(config, "dontcare"); + clientConfiguration = S3AUtils + .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_DDB); Assert.assertEquals(ddbSignerOverride, clientConfiguration.getSignerOverride()); - clientConfiguration = S3AUtils.createAwsConfForS3(config, "dontcare"); + clientConfiguration = S3AUtils + .createAwsConf(config, "dontcare", AWS_SERVICE_IDENTIFIER_S3); Assert .assertEquals(signerOverride, clientConfiguration.getSignerOverride()); } - - // Expecting generic Exception.class to handle future implementation changes. - // For now, this is an NPE - @Test(timeout = 10_000L, expected = Exception.class) - public void testCustomSignerFailureIfNotRegistered() { - Signer s1 = SignerFactory.createSigner("testsigner1", null); - } - - @Test(timeout = 10_000L) - public void testCustomSignerInitialization() { - Configuration config = new Configuration(); - SignerForTest1.reset(); - SignerForTest2.reset(); - config.set(CUSTOM_SIGNERS, "testsigner1:" + SignerForTest1.class.getName()); - initCustomSigners(config); - Signer s1 = SignerFactory.createSigner("testsigner1", null); - s1.sign(null, null); - Assert.assertEquals(true, SignerForTest1.initialized); - } - - @Test(timeout = 10_000L) - public void testMultipleCustomSignerInitialization() { - Configuration config = new Configuration(); - SignerForTest1.reset(); - SignerForTest2.reset(); - config.set(CUSTOM_SIGNERS, - "testsigner1:" + SignerForTest1.class.getName() + "," + "testsigner2:" - + SignerForTest2.class.getName()); - initCustomSigners(config); - Signer s1 = SignerFactory.createSigner("testsigner1", null); - s1.sign(null, null); - Assert.assertEquals(true, SignerForTest1.initialized); - - Signer s2 = SignerFactory.createSigner("testsigner2", null); - s2.sign(null, null); - Assert.assertEquals(true, SignerForTest2.initialized); - } - - - /** - * SignerForTest1. - */ - @Private - public static class SignerForTest1 implements Signer { - - private static boolean initialized = false; - - @Override - public void sign(SignableRequest request, AWSCredentials credentials) { - initialized = true; - } - - public static void reset() { - initialized = false; - } - } - - /** - * SignerForTest2. - */ - @Private - public static class SignerForTest2 implements Signer { - - private static boolean initialized = false; - - @Override - public void sign(SignableRequest request, AWSCredentials credentials) { - initialized = true; - } - - public static void reset() { - initialized = false; - } - } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java new file mode 100644 index 0000000000000..b55ffb1e197f1 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS; + +import com.amazonaws.SignableRequest; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.Signer; +import com.amazonaws.auth.SignerFactory; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.LambdaTestUtils; +import org.assertj.core.api.Assertions; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +public class TestSignerManager { + + @Rule + public Timeout testTimeout = new Timeout( +10_000L, TimeUnit.MILLISECONDS + ); + + @Test + public void testCustomSignerFailureIfNotRegistered() throws Exception { + LambdaTestUtils.intercept(Exception.class, + () -> SignerFactory.createSigner("testsignerUnregistered", null)); + // Expecting generic Exception.class to handle future implementation changes. + // For now, this is an NPE + } + + @Test + public void testCustomSignerInitialization() { + Configuration config = new Configuration(); + SignerForTest1.reset(); + SignerForTest2.reset(); + config.set(CUSTOM_SIGNERS, "testsigner1:" + SignerForTest1.class.getName()); + SignerManager signerManager = new SignerManager(); + signerManager.initCustomSigners(config); + Signer s1 = SignerFactory.createSigner("testsigner1", null); + s1.sign(null, null); + Assertions.assertThat(SignerForTest1.initialized) + .as(SignerForTest1.class.getName() + " not initialized") + .isEqualTo(true); + } + + @Test + public void testMultipleCustomSignerInitialization() { + Configuration config = new Configuration(); + SignerForTest1.reset(); + SignerForTest2.reset(); + config.set(CUSTOM_SIGNERS, + "testsigner1:" + SignerForTest1.class.getName() + "," + "testsigner2:" + + SignerForTest2.class.getName()); + SignerManager signerManager = new SignerManager(); + signerManager.initCustomSigners(config); + Signer s1 = SignerFactory.createSigner("testsigner1", null); + s1.sign(null, null); + Assertions.assertThat(SignerForTest1.initialized) + .as(SignerForTest1.class.getName() + " not initialized") + .isEqualTo(true); + + Signer s2 = SignerFactory.createSigner("testsigner2", null); + s2.sign(null, null); + Assertions.assertThat(SignerForTest2.initialized) + .as(SignerForTest2.class.getName() + " not initialized") + .isEqualTo(true); + } + + /** + * SignerForTest1. + */ + @Private + public static class SignerForTest1 implements Signer { + + private static boolean initialized = false; + + @Override + public void sign(SignableRequest request, AWSCredentials credentials) { + initialized = true; + } + + public static void reset() { + initialized = false; + } + } + + /** + * SignerForTest2. + */ + @Private + public static class SignerForTest2 implements Signer { + + private static boolean initialized = false; + + @Override + public void sign(SignableRequest request, AWSCredentials credentials) { + initialized = true; + } + + public static void reset() { + initialized = false; + } + } +} From 52d44c2710fa736853b33e5fbfeb6ae4d25b9dcd Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Wed, 18 Sep 2019 14:58:42 +0530 Subject: [PATCH 3/6] Checkstyle fixes. --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 25 ++++++++++--------- .../apache/hadoop/fs/s3a/SignerManager.java | 2 +- .../hadoop/fs/s3a/TestSignerManager.java | 19 ++++++++------ 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 77e4ef22279a3..0747be2d7e419 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -361,7 +361,7 @@ public void initialize(URI name, Configuration originalConf) useListV1 = (listVersion == 1); signerManager = new SignerManager(); - signerManager.initCustomSigners(conf);; + signerManager.initCustomSigners(conf); // creates the AWS client, including overriding auth chain if // the FS came with a DT diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 395773a11ddc9..b6b3ab11bddec 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -1234,22 +1234,23 @@ public static ClientConfiguration createAwsConf(Configuration conf, if (StringUtils.isNotEmpty(awsServiceIdentifier)) { String configKey = null; switch (awsServiceIdentifier) { - case AWS_SERVICE_IDENTIFIER_S3: - configKey = SIGNING_ALGORITHM_S3; - break; - case AWS_SERVICE_IDENTIFIER_DDB: - configKey = SIGNING_ALGORITHM_DDB; - break; - case AWS_SERVICE_IDENTIFIER_STS: - configKey = SIGNING_ALGORITHM_STS; - break; - default: - // Nothing to do. The original signer override is already setup + case AWS_SERVICE_IDENTIFIER_S3: + configKey = SIGNING_ALGORITHM_S3; + break; + case AWS_SERVICE_IDENTIFIER_DDB: + configKey = SIGNING_ALGORITHM_DDB; + break; + case AWS_SERVICE_IDENTIFIER_STS: + configKey = SIGNING_ALGORITHM_STS; + break; + default: + // Nothing to do. The original signer override is already setup } if (configKey != null) { String signerOverride = conf.getTrimmed(configKey, ""); if (!signerOverride.isEmpty()) { - LOG.debug("Signer override for {}} = {}", awsServiceIdentifier, signerOverride); + LOG.debug("Signer override for {}} = {}", awsServiceIdentifier, + signerOverride); awsConf.setSignerOverride(signerOverride); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java index b96fb18069898..4c3b0d9c2a9ba 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java @@ -28,7 +28,7 @@ import org.slf4j.LoggerFactory; /** - * Class to handle custom signers + * Class to handle custom signers. */ public class SignerManager implements Closeable { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java index b55ffb1e197f1..e60c2a4de366e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java @@ -18,33 +18,38 @@ package org.apache.hadoop.fs.s3a; -import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS; - import com.amazonaws.SignableRequest; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.Signer; import com.amazonaws.auth.SignerFactory; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.test.LambdaTestUtils; import org.assertj.core.api.Assertions; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.LambdaTestUtils; + +import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS; + +/** + * Tests for the SignerManager. + */ public class TestSignerManager { @Rule public Timeout testTimeout = new Timeout( -10_000L, TimeUnit.MILLISECONDS + 10_000L, TimeUnit.MILLISECONDS ); @Test public void testCustomSignerFailureIfNotRegistered() throws Exception { LambdaTestUtils.intercept(Exception.class, () -> SignerFactory.createSigner("testsignerUnregistered", null)); - // Expecting generic Exception.class to handle future implementation changes. + // Expecting generic Exception.class to handle future implementation + // changes. // For now, this is an NPE } From 3fbe3b04b8b39ed35c437912635c6ff727f7a207 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Thu, 19 Sep 2019 00:12:03 +0530 Subject: [PATCH 4/6] Removed deprecation warnings --- .../org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java | 2 +- .../src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index 4f2d731aecbbf..041b6f4c1798e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -368,7 +368,7 @@ public E expectedSessionRequestFailure( DurationInfo ignored = new DurationInfo(LOG, "requesting credentials")) { Configuration conf = new Configuration(getContract().getConf()); ClientConfiguration awsConf = - S3AUtils.createAwsConf(conf, null); + S3AUtils.createAwsConf(conf, null, AWS_SERVICE_IDENTIFIER_STS); return intercept(clazz, exceptionText, () -> { AWSSecurityTokenService tokenService = diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index e6f32af9a1fda..b9743858b211b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -680,7 +680,7 @@ public static MarshalledCredentials requestSessionCredentials( MarshalledCredentials sc = MarshalledCredentialBinding .requestSessionCredentials( buildAwsCredentialsProvider(conf), - S3AUtils.createAwsConf(conf, bucket), + S3AUtils.createAwsConf(conf, bucket, AWS_SERVICE_IDENTIFIER_STS), conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT, DEFAULT_ASSUMED_ROLE_STS_ENDPOINT), conf.getTrimmed(ASSUMED_ROLE_STS_ENDPOINT_REGION, From d1ba60e5d55a8ff204316122578f1c72f705e91b Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Thu, 19 Sep 2019 21:01:04 +0530 Subject: [PATCH 5/6] re-ordered imports --- .../main/java/org/apache/hadoop/fs/s3a/SignerManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java index 4c3b0d9c2a9ba..5ca1482b8437c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SignerManager.java @@ -17,16 +17,17 @@ */ package org.apache.hadoop.fs.s3a; -import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS; - import com.amazonaws.auth.Signer; import com.amazonaws.auth.SignerFactory; import java.io.Closeable; import java.io.IOException; -import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; + +import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS; + /** * Class to handle custom signers. */ @@ -94,6 +95,5 @@ private static void maybeRegisterSigner(String signerName, @Override public void close() throws IOException { - } } From 007360f68a2fb6b5fca128d6132783001c65eb61 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Fri, 20 Sep 2019 23:46:04 +0530 Subject: [PATCH 6/6] More import order fixes --- .../java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java | 3 ++- .../hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java | 2 +- .../apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java | 2 +- .../java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java | 3 ++- .../test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java | 3 ++- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java index d19d820bef28c..82811e625ecdb 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/STSClientFactory.java @@ -31,13 +31,14 @@ import com.amazonaws.services.securitytoken.model.Credentials; import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest; import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.s3a.Constants; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.Retries; import org.apache.hadoop.fs.s3a.S3AUtils; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java index 0f327028d4c0c..592ec61568591 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java @@ -31,12 +31,12 @@ import com.amazonaws.auth.AWSSessionCredentials; import com.amazonaws.services.securitytoken.AWSSecurityTokenService; import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.fs.s3a.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; +import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.Retries; import org.apache.hadoop.fs.s3a.S3ARetryPolicy; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java index 5f21250449633..b6ff4d982ddce 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java @@ -27,7 +27,6 @@ import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.fs.s3a.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +34,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AUtils; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 614fa53631154..32f3235e52d98 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -22,7 +22,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.S3ClientOptions; -import java.io.IOException; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.hadoop.conf.Configuration; @@ -38,6 +38,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.io.File; import java.net.URI; import java.security.PrivilegedExceptionAction; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java index e60c2a4de366e..ac759d0976a31 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSignerManager.java @@ -18,11 +18,12 @@ package org.apache.hadoop.fs.s3a; +import java.util.concurrent.TimeUnit; + import com.amazonaws.SignableRequest; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.Signer; import com.amazonaws.auth.SignerFactory; -import java.util.concurrent.TimeUnit; import org.assertj.core.api.Assertions; import org.junit.Rule; import org.junit.Test;