From 4d311676c00956a681599c01c15b54f73c11f563 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Sat, 11 Dec 2021 11:36:30 +0530 Subject: [PATCH 1/3] HADOOP-17725. Keep MSI tenant ID and client ID optional (ADDENDUM) --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 4 +-- .../fs/azurebfs/TestAccountConfiguration.java | 27 ++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 5df46eb883da1..4722d82a202d6 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -963,9 +963,9 @@ public AccessTokenProvider getTokenProvider() throws TokenAccessProviderExceptio FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT, AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT); String tenantGuid = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT); + getPasswordString(FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT); String clientId = - getMandatoryPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); + getPasswordString(FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID); String authority = getTrimmedPasswordString( FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY, AuthConfigurations.DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java index 17da772d0819b..1928ada82ef13 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.TokenAccessProviderException; +import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider; import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider; import org.apache.hadoop.fs.azurebfs.oauth2.CustomTokenProviderAdapter; import org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider; @@ -65,7 +66,7 @@ * that do allow default values (all others) follow another form. */ public class TestAccountConfiguration { - private static final String TEST_OAUTH_PROVIDER_CLASS_CONFIG = "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"; + private static final String TEST_OAUTH_MSI_TOKEN_PROVIDER_CLASS_CONFIG = "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider"; private static final String TEST_CUSTOM_PROVIDER_CLASS_CONFIG = "org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider"; private static final String TEST_SAS_PROVIDER_CLASS_CONFIG_1 = "org.apache.hadoop.fs.azurebfs.extensions.MockErrorSASTokenProvider"; private static final String TEST_SAS_PROVIDER_CLASS_CONFIG_2 = "org.apache.hadoop.fs.azurebfs.extensions.MockSASTokenProvider"; @@ -444,6 +445,30 @@ private static void testMissingConfigKey(final AbfsConfiguration abfsConf, () -> abfsConf.getTokenProvider().getClass().getTypeName()))); } + @Test + public void testClientAndTenantIdOptionalWhenUsingMsiTokenProvider() throws Throwable { + final String accountName = "account"; + final Configuration conf = new Configuration(); + final AbfsConfiguration abfsConf = new AbfsConfiguration(conf, accountName); + + final String accountNameSuffix = "." + abfsConf.getAccountName(); + String authKey = FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME + accountNameSuffix; + String providerClassKey = ""; + String providerClassValue = ""; + + providerClassKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountNameSuffix; + providerClassValue = TEST_OAUTH_MSI_TOKEN_PROVIDER_CLASS_CONFIG; + + abfsConf.set(authKey, AuthType.OAuth.toString()); + abfsConf.set(providerClassKey, providerClassValue); + + AccessTokenProvider tokenProviderTypeName = abfsConf.getTokenProvider(); + // Test that we managed to instantiate an MsiTokenProvider without having to define the tenant and client ID. + // Those 2 fields are optional as they can automatically be determined by the Azure Metadata service when + // running on an Azure VM. + Assertions.assertThat(tokenProviderTypeName).isInstanceOf(MsiTokenProvider.class); + } + public void testGlobalAndAccountOAuthPrecedence(AbfsConfiguration abfsConf, AuthType globalAuthType, AuthType accountSpecificAuthType) From a602b81fc2638a75a52df220773e9de9a3c65550 Mon Sep 17 00:00:00 2001 From: Carl Levasseur Date: Fri, 10 Mar 2023 11:19:12 +0100 Subject: [PATCH 2/3] test disabled --- .../hadoop/fs/azurebfs/TestAccountConfiguration.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java index 1928ada82ef13..8558c8ed1c77b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java @@ -66,6 +66,7 @@ * that do allow default values (all others) follow another form. */ public class TestAccountConfiguration { + private static final String TEST_OAUTH_PROVIDER_CLASS_CONFIG = "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"; private static final String TEST_OAUTH_MSI_TOKEN_PROVIDER_CLASS_CONFIG = "org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider"; private static final String TEST_CUSTOM_PROVIDER_CLASS_CONFIG = "org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider"; private static final String TEST_SAS_PROVIDER_CLASS_CONFIG_1 = "org.apache.hadoop.fs.azurebfs.extensions.MockErrorSASTokenProvider"; @@ -91,11 +92,6 @@ public class TestAccountConfiguration { FS_AZURE_ACCOUNT_OAUTH_USER_NAME, FS_AZURE_ACCOUNT_OAUTH_USER_PASSWORD)); - private static final List MSI_TOKEN_OAUTH_CONFIG_KEYS = - Collections.unmodifiableList(Arrays.asList( - FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT, - FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID)); - private static final List REFRESH_TOKEN_OAUTH_CONFIG_KEYS = Collections.unmodifiableList(Arrays.asList( FS_AZURE_ACCOUNT_OAUTH_REFRESH_TOKEN, @@ -411,10 +407,8 @@ public void testAccessTokenProviderPrecedence() public void testOAuthConfigPropNotFound() throws Throwable { testConfigPropNotFound(CLIENT_CREDENTIAL_OAUTH_CONFIG_KEYS, ClientCredsTokenProvider.class.getName()); testConfigPropNotFound(USER_PASSWORD_OAUTH_CONFIG_KEYS, UserPasswordTokenProvider.class.getName()); - testConfigPropNotFound(MSI_TOKEN_OAUTH_CONFIG_KEYS, MsiTokenProvider.class.getName()); testConfigPropNotFound(REFRESH_TOKEN_OAUTH_CONFIG_KEYS, RefreshTokenBasedTokenProvider.class.getName()); testConfigPropNotFound(WORKLOAD_IDENTITY_OAUTH_CONFIG_KEYS, WorkloadIdentityTokenProvider.class.getName()); - } private void testConfigPropNotFound(List configKeys, From b32b2218e70a8ec22762ad61e61912dd6a988823 Mon Sep 17 00:00:00 2001 From: Carl Levasseur Date: Wed, 24 Jul 2024 10:12:55 +0200 Subject: [PATCH 3/3] address comment about test description --- .../org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java index 8558c8ed1c77b..483a7e3d5d58e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAccountConfiguration.java @@ -460,7 +460,7 @@ public void testClientAndTenantIdOptionalWhenUsingMsiTokenProvider() throws Thro // Test that we managed to instantiate an MsiTokenProvider without having to define the tenant and client ID. // Those 2 fields are optional as they can automatically be determined by the Azure Metadata service when // running on an Azure VM. - Assertions.assertThat(tokenProviderTypeName).isInstanceOf(MsiTokenProvider.class); + Assertions.assertThat(tokenProviderTypeName).describedAs("Token Provider Should be MsiTokenProvider").isInstanceOf(MsiTokenProvider.class); } public void testGlobalAndAccountOAuthPrecedence(AbfsConfiguration abfsConf,