From 433ce77590cc6cfafc4d2d5c4de782087a30b714 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 18 Oct 2023 15:48:31 +0100 Subject: [PATCH 1/5] HADOOP-18945. S3A: IAMInstanceCredentialsProvider failing. - Enable async refresh in inner credential provider. - Store provider instance in a field; use synchronized() call to handle failure and switch from container to EC2 provider. - Close inner provider in close(); - use ErrorTranslation to extract any IOE at base on stack - rename maybeExtractNetworkException to maybeExtractIOException (as requested in review of third party support) - test for error translation going through UncheckedIOException - TestIAMInstanceCredentialsProvider to test fallback process TestIAMInstanceCredentialsProvider is tricky as there are different test outcomes 1. In EC2: credentials resolved. Assert comes with a key. 2. Not in EC2: network error trying to talk to the service. Assert wrapped exception is an IOE. 3. IMDS resolution disabled by env var/sysprop. Expect the message to contain the text from the SDK This is potentially brittle; there may be followups. Change-Id: I1ad1224e31c0c3646e07a93d503a84885bf99ebe --- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 6 +- .../auth/CredentialProviderListFactory.java | 5 +- .../auth/IAMInstanceCredentialsProvider.java | 69 ++++++++++-- .../hadoop/fs/s3a/impl/ErrorTranslation.java | 6 +- .../TestIAMInstanceCredentialsProvider.java | 102 ++++++++++++++++++ .../fs/s3a/impl/TestErrorTranslation.java | 31 +++++- 6 files changed, 197 insertions(+), 22 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 6798a99c19e1e..ef71228c77db2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -80,7 +80,7 @@ import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf; import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unsupportedConstructor; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.*; -import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException; +import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -194,7 +194,7 @@ public static IOException translateException(@Nullable String operation, return ioe; } // network problems covered by an IOE inside the exception chain. - ioe = maybeExtractNetworkException(path, exception); + ioe = maybeExtractIOException(path, exception); if (ioe != null) { return ioe; } @@ -583,7 +583,7 @@ public static long dateToLong(final Date date) { * @param uri URI of the FS * @param interfaceImplemented interface that this class implements * @param methodName name of factory method to be invoked - * @param configKey config key under which this class is specified + * @param configKey config key under which this class is specified; used for exception text * @param Instance of class * @return instance of the specified class * @throws IOException on any problem diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java index b106777dd29cc..8a80668265961 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -287,12 +287,13 @@ public static AWSCredentialProviderList buildAWSProviderList( * @param conf configuration * @param className credential class name * @param uri URI of the FS - * @param key configuration key to use + * @param key configuration key used; used for exception text * @return the instantiated class * @throws IOException on any instantiation failure. * @see S3AUtils#getInstanceFromReflection */ - private static AwsCredentialsProvider createAWSV2CredentialProvider(Configuration conf, + @VisibleForTesting + public static AwsCredentialsProvider createAWSV2CredentialProvider(Configuration conf, String className, @Nullable URI uri, final String key) throws IOException { LOG.debug("Credential provider class is {}", className); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java index 2e39b275b4a4d..dad3b4f28776c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java @@ -21,15 +21,20 @@ import java.io.Closeable; import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; +import software.amazon.awssdk.auth.credentials.HttpCredentialsProvider; import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; import software.amazon.awssdk.core.exception.SdkClientException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException; + /** * This is an IAM credential provider which wraps * an {@code ContainerCredentialsProvider} @@ -48,10 +53,18 @@ public class IAMInstanceCredentialsProvider implements AwsCredentialsProvider, Closeable { - private final AwsCredentialsProvider containerCredentialsProvider = - ContainerCredentialsProvider.builder().build(); + private static final Logger LOG = + LoggerFactory.getLogger(IAMInstanceCredentialsProvider.class); + + private HttpCredentialsProvider iamCredentialsProvider = null; + + private boolean isContainerCredentialsProvider; public IAMInstanceCredentialsProvider() { + isContainerCredentialsProvider = true; + iamCredentialsProvider = ContainerCredentialsProvider.builder() + .asyncCredentialUpdateEnabled(true) + .build(); } /** @@ -65,9 +78,16 @@ public AwsCredentials resolveCredentials() { try { return getCredentials(); } catch (SdkClientException e) { + + // if the exception contains an IOE, extract it + // so its type is the immediate cause of this new exception. + Throwable t = e; + final IOException ioe = maybeExtractIOException("IAM endpoint", e); + if (ioe != null) { + t = ioe; + } throw new NoAwsCredentialsException("IAMInstanceCredentialsProvider", - e.getMessage(), - e); + e.getMessage(), t); } } @@ -78,23 +98,52 @@ public AwsCredentials resolveCredentials() { * * @return credentials */ - private AwsCredentials getCredentials() { + private synchronized AwsCredentials getCredentials() { try { - return containerCredentialsProvider.resolveCredentials(); + return iamCredentialsProvider.resolveCredentials(); } catch (SdkClientException e) { - return InstanceProfileCredentialsProvider.create().resolveCredentials(); + LOG.debug("Failed to get credentials from container provider,", e); + if (isContainerCredentialsProvider) { + // create instance profile provider + LOG.debug("Switching to instance provider", e); + + // close it to shut down any thread + iamCredentialsProvider.close(); + isContainerCredentialsProvider = false; + iamCredentialsProvider = InstanceProfileCredentialsProvider.builder() + .asyncCredentialUpdateEnabled(true) + .build(); + return iamCredentialsProvider.resolveCredentials(); + } else { + // already using instance profile provider, so fail + throw e; + } + } } + /** + * Is this a container credentials provider? + * @return true if the container credentials provider is in use; + * false for InstanceProfileCredentialsProvider + */ + public boolean isContainerCredentialsProvider() { + return isContainerCredentialsProvider; + } + @Override - public void close() throws IOException { - // no-op. + public synchronized void close() throws IOException { + // this be true but just for safety... + if (iamCredentialsProvider != null) { + iamCredentialsProvider.close(); + } } @Override public String toString() { return "IAMInstanceCredentialsProvider{" + - "containerCredentialsProvider=" + containerCredentialsProvider + + "credentialsProvider=" + iamCredentialsProvider + + ", isContainerCredentialsProvider=" + isContainerCredentialsProvider + '}'; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java index 7b5190becc487..f8a1f907bb3b1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java @@ -79,7 +79,7 @@ public static boolean isObjectNotFound(AwsServiceException e) { * @param thrown exception * @return a translated exception or null. */ - public static IOException maybeExtractNetworkException(String path, Throwable thrown) { + public static IOException maybeExtractIOException(String path, Throwable thrown) { if (thrown == null) { return null; @@ -100,7 +100,9 @@ public static IOException maybeExtractNetworkException(String path, Throwable th // as a new instance is created through reflection, the // class of the returned instance will be that of the innermost, // unless no suitable constructor is available. - return wrapWithInnerIOE(path, thrown, (IOException) cause); + final IOException ioe = (IOException) cause; + + return wrapWithInnerIOE(path, thrown, ioe); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java new file mode 100644 index 0000000000000..8bd617f73d8d1 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.auth; + +import java.io.IOException; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentials; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + + +/** + * Unit tests for IAMInstanceCredentials provider. + * This is a bit tricky as don't want to + */ +public class TestIAMInstanceCredentialsProvider extends AbstractHadoopTestBase { + + private static final Logger LOG = + LoggerFactory.getLogger(TestIAMInstanceCredentialsProvider.class); + + /** + * Error string from + * software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider, + * if IAM resolution has been disabled. + */ + public static final String DISABLED = + "IMDS credentials have been disabled by environment variable or system property"; + + /** + * Test an immediate create/close. + */ + @Test + public void testIAMInstanceCredentialsProviderClose() throws Throwable { + new IAMInstanceCredentialsProvider().close(); + } + + /** + * Test instantiation. + * Multiple outcomes depending on host setup. + *
    + *
  1. In EC2: credentials resolved + * Assert comes with a key.
    2. + *
  2. Not in EC2: network error trying to talk to the service. + * Assert wrapped exception is an IOE.
    3. + *
  3. IMDS resolution disabled by env var/sysprop. + * Expect the message to contain the "disabled" text.
    4. + *
+ */ + @Test + public void testIAMInstanceCredentialsInstantiate() throws Throwable { + try (IAMInstanceCredentialsProvider provider = new IAMInstanceCredentialsProvider()) { + try { + final AwsCredentials credentials = provider.resolveCredentials(); + // if we get here this test suite is running in a container/EC2 + LOG.info("Credentials: retrieved from {}: key={}", + provider.isContainerCredentialsProvider() ? "container" : "EC2", + credentials.accessKeyId()); + Assertions.assertThat(credentials.accessKeyId()) + .describedAs("Access key from IMDS") + .isNotBlank(); + } catch (NoAwsCredentialsException expected) { + // this is expected if the test is not running in a container/EC2 + LOG.info("Not running in a container/EC2"); + LOG.info("Exception raised", expected); + // and we expect to have fallen back to the EC2 provider + Assertions.assertThat(provider.isContainerCredentialsProvider()) + .describedAs("%s: shoud be using IAM credentials provider") + .isFalse(); + final Throwable cause = expected.getCause(); + if (cause == null) { + throw expected; + } + if (!(cause instanceof IOException) + && !cause.toString().contains(DISABLED)) { + throw new AssertionError("Cause not a IOException", cause); + } + } + } + } + + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java index a1fdbabb94793..0f0b2c0c34bb5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java @@ -19,8 +19,10 @@ package org.apache.hadoop.fs.s3a.impl; import java.io.IOException; +import java.io.UncheckedIOException; import java.net.ConnectException; import java.net.NoRouteToHostException; +import java.net.SocketTimeoutException; import java.net.UnknownHostException; import java.util.Collections; @@ -31,9 +33,10 @@ import software.amazon.awssdk.core.retry.RetryPolicyContext; import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException; import org.apache.hadoop.test.AbstractHadoopTestBase; -import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractNetworkException; +import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assert.assertTrue; @@ -64,7 +67,7 @@ public void testUnknownHostExceptionExtraction() throws Throwable { new UnknownHostException("bottom"))); final IOException ioe = intercept(UnknownHostException.class, "top", () -> { - throw maybeExtractNetworkException("", thrown); + throw maybeExtractIOException("", thrown); }); // the wrapped exception is the top level one: no stack traces have @@ -79,7 +82,7 @@ public void testUnknownHostExceptionExtraction() throws Throwable { public void testNoRouteToHostExceptionExtraction() throws Throwable { intercept(NoRouteToHostException.class, "top", () -> { - throw maybeExtractNetworkException("p2", + throw maybeExtractIOException("p2", sdkException("top", sdkException("middle", new NoRouteToHostException("bottom")))); @@ -90,17 +93,35 @@ public void testNoRouteToHostExceptionExtraction() throws Throwable { public void testConnectExceptionExtraction() throws Throwable { intercept(ConnectException.class, "top", () -> { - throw maybeExtractNetworkException("p1", + throw maybeExtractIOException("p1", sdkException("top", sdkException("middle", new ConnectException("bottom")))); }); } + + /** + * When there is an UncheckedIOException, its inner class is + * extracted. + */ + @Test + public void testUncheckedIOExceptionExtraction() throws Throwable { + intercept(SocketTimeoutException.class, "top", + () -> { + final SdkClientException thrown = sdkException("top", + sdkException("middle", + new UncheckedIOException( + new SocketTimeoutException("bottom")))); + throw maybeExtractIOException("p1", + new NoAwsCredentialsException("IamProvider", thrown.toString(), thrown)); + }); + } + @Test public void testNoConstructorExtraction() throws Throwable { intercept(PathIOException.class, NoConstructorIOE.MESSAGE, () -> { - throw maybeExtractNetworkException("p1", + throw maybeExtractIOException("p1", sdkException("top", sdkException("middle", new NoConstructorIOE()))); From 8d3bff91f26724028db9cd564494fc8084b90437 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 18 Oct 2023 17:01:28 +0100 Subject: [PATCH 2/5] HADOOP-18945. S3A: IAMInstanceCredentialsProvider failing. Minor tweaks to test, javadocs, revert some changes added for testing but which aren't used -just cuts down the patch size Change-Id: Id39f1196095a2157dbf61291285885542a44a782 --- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 2 +- .../auth/CredentialProviderListFactory.java | 5 ++-- .../auth/IAMInstanceCredentialsProvider.java | 27 ++++++++++++++++--- .../TestIAMInstanceCredentialsProvider.java | 3 +++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index ef71228c77db2..7466690744e82 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -583,7 +583,7 @@ public static long dateToLong(final Date date) { * @param uri URI of the FS * @param interfaceImplemented interface that this class implements * @param methodName name of factory method to be invoked - * @param configKey config key under which this class is specified; used for exception text + * @param configKey config key under which this class is specified * @param Instance of class * @return instance of the specified class * @throws IOException on any problem diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java index 8a80668265961..b106777dd29cc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java @@ -287,13 +287,12 @@ public static AWSCredentialProviderList buildAWSProviderList( * @param conf configuration * @param className credential class name * @param uri URI of the FS - * @param key configuration key used; used for exception text + * @param key configuration key to use * @return the instantiated class * @throws IOException on any instantiation failure. * @see S3AUtils#getInstanceFromReflection */ - @VisibleForTesting - public static AwsCredentialsProvider createAWSV2CredentialProvider(Configuration conf, + private static AwsCredentialsProvider createAWSV2CredentialProvider(Configuration conf, String className, @Nullable URI uri, final String key) throws IOException { LOG.debug("Credential provider class is {}", className); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java index dad3b4f28776c..080b79e7f20d5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java @@ -41,12 +41,18 @@ * to provide credentials when the S3A connector is instantiated on AWS EC2 * or the AWS container services. *

- * When it fails to authenticate, it raises a - * {@link NoAwsCredentialsException} which can be recognized by retry handlers + * The provider is initialized with async credential refresh enabled to be less + * brittle against transient network issues. + *

+ * If the ContainerCredentialsProvider fails to authenticate, then an instance of + * {@link InstanceProfileCredentialsProvider} is created and attemped to + * be used instead, again with async credential refresh enabled. + *

+ * If both credential providers fail, a {@link NoAwsCredentialsException} + * is thrown, which can be recognized by retry handlers * as a non-recoverable failure. *

* It is implicitly public; marked evolving as we can change its semantics. - * */ @InterfaceAudience.Public @InterfaceStability.Evolving @@ -56,10 +62,23 @@ public class IAMInstanceCredentialsProvider private static final Logger LOG = LoggerFactory.getLogger(IAMInstanceCredentialsProvider.class); - private HttpCredentialsProvider iamCredentialsProvider = null; + /** + * The credentials provider. + * Initially a container credentials provider, but if that fails + * fall back to the instance profile provider. + */ + private HttpCredentialsProvider iamCredentialsProvider; + /** + * Is the container credentials provider in use? + */ private boolean isContainerCredentialsProvider; + /** + * Constructor. + * Build credentials provider with async refresh, + * mark {@link #isContainerCredentialsProvider} as true. + */ public IAMInstanceCredentialsProvider() { isContainerCredentialsProvider = true; iamCredentialsProvider = ContainerCredentialsProvider.builder() diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java index 8bd617f73d8d1..529cc6700c7ae 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java @@ -78,6 +78,9 @@ public void testIAMInstanceCredentialsInstantiate() throws Throwable { Assertions.assertThat(credentials.accessKeyId()) .describedAs("Access key from IMDS") .isNotBlank(); + + // and if we get here, so does a second call + final AwsCredentials credentials2 = provider.resolveCredentials(); } catch (NoAwsCredentialsException expected) { // this is expected if the test is not running in a container/EC2 LOG.info("Not running in a container/EC2"); From 922f82a9459f279bde0d6f6cb1268bcd12dbb423 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 19 Oct 2023 17:07:09 +0100 Subject: [PATCH 3/5] HADOOP-18945. Review comments Change-Id: I6241091ace05c51fc26e42b99fdb00b13e161a69 --- .../auth/TestIAMInstanceCredentialsProvider.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java index 529cc6700c7ae..a20676712c85c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java @@ -41,7 +41,7 @@ public class TestIAMInstanceCredentialsProvider extends AbstractHadoopTestBase { /** * Error string from * software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider, - * if IAM resolution has been disabled. + * if IAM resolution has been disabled: {@value}. */ public static final String DISABLED = "IMDS credentials have been disabled by environment variable or system property"; @@ -58,12 +58,14 @@ public void testIAMInstanceCredentialsProviderClose() throws Throwable { * Test instantiation. * Multiple outcomes depending on host setup. *

    - *
  1. In EC2: credentials resolved - * Assert comes with a key.
    2. - *
  2. Not in EC2: network error trying to talk to the service. + *
  3. In EC2: credentials resolved. + * Assert the credentials comes with a key.
    4. + *
  4. Not in EC2: NoAwsCredentialsException wraps network error trying + * to talk to the service. * Assert wrapped exception is an IOE.
    5. *
  5. IMDS resolution disabled by env var/sysprop. - * Expect the message to contain the "disabled" text.
    6. + * NoAwsCredentialsException raised doesn't contain an IOE. + * Require the message to contain the {@link #DISABLED} text.j *
*/ @Test @@ -85,9 +87,9 @@ public void testIAMInstanceCredentialsInstantiate() throws Throwable { // this is expected if the test is not running in a container/EC2 LOG.info("Not running in a container/EC2"); LOG.info("Exception raised", expected); - // and we expect to have fallen back to the EC2 provider + // and we expect to have fallen back to InstanceProfileCredentialsProvider Assertions.assertThat(provider.isContainerCredentialsProvider()) - .describedAs("%s: shoud be using IAM credentials provider") + .describedAs("%s: shoud be using InstanceProfileCredentialsProvider") .isFalse(); final Throwable cause = expected.getCause(); if (cause == null) { From 55297c1ae1807326e313642e6651d6348b93b93e Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 19 Oct 2023 17:09:32 +0100 Subject: [PATCH 4/5] HADOOP-18945. IDE warns of unused variable Change-Id: Iac2f9821e1f5d597313145bc63d1467cd68ca6d6 --- .../hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java index a20676712c85c..9d84ddda4aafe 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java @@ -82,7 +82,7 @@ public void testIAMInstanceCredentialsInstantiate() throws Throwable { .isNotBlank(); // and if we get here, so does a second call - final AwsCredentials credentials2 = provider.resolveCredentials(); + provider.resolveCredentials(); } catch (NoAwsCredentialsException expected) { // this is expected if the test is not running in a container/EC2 LOG.info("Not running in a container/EC2"); From 9f2ffea82c827e4eb58b80346406a30a01b89d39 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 20 Oct 2023 16:37:29 +0100 Subject: [PATCH 5/5] HADOOP-18945. incomplete javadoc comment Change-Id: I0135620db5228c8aa3ecac03a0e7797984f872c9 --- .../fs/s3a/auth/TestIAMInstanceCredentialsProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java index 9d84ddda4aafe..c8986eab9b850 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java @@ -28,10 +28,10 @@ import org.apache.hadoop.test.AbstractHadoopTestBase; - /** * Unit tests for IAMInstanceCredentials provider. - * This is a bit tricky as don't want to + * This is a bit tricky as we don't want to require running in EC2, + * but nor do we want a test which doesn't work in EC2. */ public class TestIAMInstanceCredentialsProvider extends AbstractHadoopTestBase {