diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java index 2170550dde..b2eb8fcfa5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java @@ -66,9 +66,9 @@ import org.apache.polaris.core.policy.PolicyEntity; import org.apache.polaris.core.policy.PolicyMappingUtil; import org.apache.polaris.core.policy.PolicyType; -import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1605,7 +1605,7 @@ private void revokeGrantRecord( PolarisStorageConfigurationInfo storageConfigurationInfo = BaseMetaStoreManager.extractStorageConfiguration(callCtx, reloadedEntity.getEntity()); try { - EnumMap creds = + EnumMap creds = storageIntegration.getSubscopedCreds( callCtx.getDiagServices(), storageConfigurationInfo, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ScopedCredentialsResult.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ScopedCredentialsResult.java index 2de1609d7e..60e1cee01a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ScopedCredentialsResult.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/dao/entity/ScopedCredentialsResult.java @@ -24,13 +24,13 @@ import jakarta.annotation.Nullable; import java.util.EnumMap; import java.util.Map; -import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageAccessProperty; /** Result of a getSubscopedCredsForEntity() call */ public class ScopedCredentialsResult extends BaseResult { // null if not success. Else, set of name/value pairs for the credentials - private final EnumMap credentials; + private final EnumMap credentials; /** * Constructor for an error @@ -49,7 +49,7 @@ public ScopedCredentialsResult( * * @param credentials credentials */ - public ScopedCredentialsResult(@Nonnull EnumMap credentials) { + public ScopedCredentialsResult(@Nonnull EnumMap credentials) { super(ReturnStatus.SUCCESS); this.credentials = credentials; } @@ -60,13 +60,13 @@ private ScopedCredentialsResult( @JsonProperty("extraInformation") String extraInformation, @JsonProperty("credentials") Map credentials) { super(returnStatus, extraInformation); - this.credentials = new EnumMap<>(PolarisCredentialProperty.class); + this.credentials = new EnumMap<>(StorageAccessProperty.class); if (credentials != null) { - credentials.forEach((k, v) -> this.credentials.put(PolarisCredentialProperty.valueOf(k), v)); + credentials.forEach((k, v) -> this.credentials.put(StorageAccessProperty.valueOf(k), v)); } } - public EnumMap getCredentials() { + public EnumMap getCredentials() { return credentials; } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index 9f508eb4e1..6eb48c12ef 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -67,9 +67,9 @@ import org.apache.polaris.core.policy.PolicyEntity; import org.apache.polaris.core.policy.PolicyMappingUtil; import org.apache.polaris.core.policy.PolicyType; -import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -2035,7 +2035,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( PolarisStorageConfigurationInfo storageConfigurationInfo = BaseMetaStoreManager.extractStorageConfiguration(callCtx, reloadedEntity.getEntity()); try { - EnumMap creds = + EnumMap creds = storageIntegration.getSubscopedCreds( callCtx.getDiagServices(), storageConfigurationInfo, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java new file mode 100644 index 0000000000..61a754eccd --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage; + +import java.util.Map; +import org.apache.polaris.immutables.PolarisImmutable; + +@PolarisImmutable +public interface AccessConfig { + Map credentials(); + + Map extraProperties(); + + static ImmutableAccessConfig.Builder builder() { + return ImmutableAccessConfig.builder(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java index eec9a42094..5f100a943b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageIntegration.java @@ -54,7 +54,7 @@ public String getStorageIdentifierOrId() { * @param allowedWriteLocations a set of allowed to write locations * @return An enum map including the scoped credentials */ - public abstract EnumMap getSubscopedCreds( + public abstract EnumMap getSubscopedCreds( @Nonnull PolarisDiagnostics diagnostics, @Nonnull T storageConfig, boolean allowListOperation, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisCredentialProperty.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java similarity index 76% rename from polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisCredentialProperty.java rename to polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java index 2f21a84fd7..bfd8c934d3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisCredentialProperty.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java @@ -18,8 +18,13 @@ */ package org.apache.polaris.core.storage; -/** Enum of polaris supported credential properties */ -public enum PolarisCredentialProperty { +/** + * A subset of Iceberg catalog properties recognized by Polaris. + * + *

Most of these properties are meant to configure Iceberg FileIO objects for accessing data in + * storage. + */ +public enum StorageAccessProperty { AWS_KEY_ID(String.class, "s3.access-key-id", "the aws access key id"), AWS_SECRET_KEY(String.class, "s3.secret-access-key", "the aws access key secret"), AWS_TOKEN(String.class, "s3.session-token", "the aws scoped access token"), @@ -27,6 +32,9 @@ public enum PolarisCredentialProperty { String.class, "s3.session-token-expires-at-ms", "the time the aws session token expires, in milliseconds"), + AWS_ENDPOINT(String.class, "s3.endpoint", "the S3 endpoint to use for requests", false), + AWS_PATH_STYLE_ACCESS( + Boolean.class, "s3.path-style-access", "whether to use S3 path style access", false), CLIENT_REGION( String.class, "client.region", "region to configure client for making requests to AWS"), @@ -50,19 +58,30 @@ public enum PolarisCredentialProperty { private final Class valueType; private final String propertyName; private final String description; + private final boolean isCredential; /* s3.access-key-id`: id for for credentials that provide access to the data in S3 - `s3.secret-access-key`: secret for credentials that provide access to data in S3 - `s3.session-token */ - PolarisCredentialProperty(Class valueType, String propertyName, String description) { + StorageAccessProperty(Class valueType, String propertyName, String description) { + this(valueType, propertyName, description, true); + } + + StorageAccessProperty( + Class valueType, String propertyName, String description, boolean isCredential) { this.valueType = valueType; this.propertyName = propertyName; this.description = description; + this.isCredential = isCredential; } public String getPropertyName() { return propertyName; } + + public boolean isCredential() { + return isCredential; + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index 9b1c64900b..0fc72ac01a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -31,7 +31,7 @@ import java.util.stream.Stream; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.storage.InMemoryStorageIntegration; -import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.StorageUtil; import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; import software.amazon.awssdk.policybuilder.iam.IamEffect; @@ -54,7 +54,7 @@ public AwsCredentialsStorageIntegration(StsClient stsClient) { /** {@inheritDoc} */ @Override - public EnumMap getSubscopedCreds( + public EnumMap getSubscopedCreds( @Nonnull PolarisDiagnostics diagnostics, @Nonnull AwsStorageConfigurationInfo storageConfig, boolean allowListOperation, @@ -75,28 +75,28 @@ public EnumMap getSubscopedCreds( .toJson()) .durationSeconds(loadConfig(STORAGE_CREDENTIAL_DURATION_SECONDS)) .build()); - EnumMap credentialMap = - new EnumMap<>(PolarisCredentialProperty.class); - credentialMap.put(PolarisCredentialProperty.AWS_KEY_ID, response.credentials().accessKeyId()); + EnumMap credentialMap = + new EnumMap<>(StorageAccessProperty.class); + credentialMap.put(StorageAccessProperty.AWS_KEY_ID, response.credentials().accessKeyId()); credentialMap.put( - PolarisCredentialProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); - credentialMap.put(PolarisCredentialProperty.AWS_TOKEN, response.credentials().sessionToken()); + StorageAccessProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); + credentialMap.put(StorageAccessProperty.AWS_TOKEN, response.credentials().sessionToken()); Optional.ofNullable(response.credentials().expiration()) .ifPresent( i -> { credentialMap.put( - PolarisCredentialProperty.EXPIRATION_TIME, String.valueOf(i.toEpochMilli())); + StorageAccessProperty.EXPIRATION_TIME, String.valueOf(i.toEpochMilli())); credentialMap.put( - PolarisCredentialProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, + StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, String.valueOf(i.toEpochMilli())); }); if (storageConfig.getRegion() != null) { - credentialMap.put(PolarisCredentialProperty.CLIENT_REGION, storageConfig.getRegion()); + credentialMap.put(StorageAccessProperty.CLIENT_REGION, storageConfig.getRegion()); } if (storageConfig.getAwsPartition().equals("aws-us-gov") - && credentialMap.get(PolarisCredentialProperty.CLIENT_REGION) == null) { + && credentialMap.get(StorageAccessProperty.CLIENT_REGION) == null) { throw new IllegalArgumentException( String.format( "AWS region must be set when using partition %s", storageConfig.getAwsPartition())); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java index 62e4fc4dc1..bcbc91a5c8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java @@ -48,7 +48,7 @@ import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.storage.InMemoryStorageIntegration; -import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Mono; @@ -70,14 +70,14 @@ public AzureCredentialsStorageIntegration() { } @Override - public EnumMap getSubscopedCreds( + public EnumMap getSubscopedCreds( @Nonnull PolarisDiagnostics diagnostics, @Nonnull AzureStorageConfigurationInfo storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, @Nonnull Set allowedWriteLocations) { - EnumMap credentialMap = - new EnumMap<>(PolarisCredentialProperty.class); + EnumMap credentialMap = + new EnumMap<>(StorageAccessProperty.class); String loc = !allowedWriteLocations.isEmpty() ? allowedWriteLocations.stream().findAny().orElse(null) @@ -170,10 +170,10 @@ public EnumMap getSubscopedCreds( throw new RuntimeException( String.format("Endpoint %s not supported", location.getEndpoint())); } - credentialMap.put(PolarisCredentialProperty.AZURE_SAS_TOKEN, sasToken); - credentialMap.put(PolarisCredentialProperty.AZURE_ACCOUNT_HOST, storageDnsName); + credentialMap.put(StorageAccessProperty.AZURE_SAS_TOKEN, sasToken); + credentialMap.put(StorageAccessProperty.AZURE_ACCOUNT_HOST, storageDnsName); credentialMap.put( - PolarisCredentialProperty.EXPIRATION_TIME, + StorageAccessProperty.EXPIRATION_TIME, String.valueOf(sanitizedEndTime.toInstant().toEpochMilli())); return credentialMap; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java index 3fe5931950..ef92973600 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java @@ -23,6 +23,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.annotations.VisibleForTesting; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import java.time.Duration; import java.util.Map; import java.util.Optional; @@ -35,6 +36,7 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult; +import org.apache.polaris.core.storage.AccessConfig; import org.apache.polaris.core.storage.PolarisCredentialVendor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,7 +101,7 @@ private static long maxCacheDurationMs() { * @param allowedWriteLocations a set of allowed to write locations. * @return the a map of string containing the scoped creds information */ - public Map getOrGenerateSubScopeCreds( + public AccessConfig getOrGenerateSubScopeCreds( @Nonnull PolarisCredentialVendor credentialVendor, @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntity polarisEntity, @@ -142,13 +144,19 @@ public Map getOrGenerateSubScopeCreds( "Failed to get subscoped credentials: %s", scopedCredentialsResult.getExtraInformation()); }; - return cache.get(key, loader).convertToMapOfString(); + return cache.get(key, loader).toAccessConfig(); } - public Map getIfPresent(StorageCredentialCacheKey key) { + @VisibleForTesting + @Nullable + Map getIfPresent(StorageCredentialCacheKey key) { + return getAccessConfig(key).map(AccessConfig::credentials).orElse(null); + } + + @VisibleForTesting + Optional getAccessConfig(StorageCredentialCacheKey key) { return Optional.ofNullable(cache.getIfPresent(key)) - .map(StorageCredentialCacheEntry::convertToMapOfString) - .orElse(null); + .map(StorageCredentialCacheEntry::toAccessConfig); } private boolean isTypeSupported(PolarisEntityType type) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheEntry.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheEntry.java index 2649ee99c3..51fa85a42c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheEntry.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheEntry.java @@ -19,16 +19,17 @@ package org.apache.polaris.core.storage.cache; import java.util.EnumMap; -import java.util.HashMap; -import java.util.Map; +import java.util.function.BiConsumer; import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult; -import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.AccessConfig; +import org.apache.polaris.core.storage.ImmutableAccessConfig; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.azure.AzureLocation; /** A storage credential cached entry. */ public class StorageCredentialCacheEntry { /** The scoped creds map that is fetched from a creds vending service */ - public final EnumMap credsMap; + public final EnumMap credsMap; private final ScopedCredentialsResult scopedCredentialsResult; @@ -39,15 +40,14 @@ public StorageCredentialCacheEntry(ScopedCredentialsResult scopedCredentialsResu /** Get the expiration time in millisecond for the cached entry */ public long getExpirationTime() { - if (credsMap.containsKey(PolarisCredentialProperty.GCS_ACCESS_TOKEN_EXPIRES_AT)) { - return Long.parseLong(credsMap.get(PolarisCredentialProperty.GCS_ACCESS_TOKEN_EXPIRES_AT)); + if (credsMap.containsKey(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT)) { + return Long.parseLong(credsMap.get(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT)); } - if (credsMap.containsKey(PolarisCredentialProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS)) { - return Long.parseLong( - credsMap.get(PolarisCredentialProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS)); + if (credsMap.containsKey(StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS)) { + return Long.parseLong(credsMap.get(StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS)); } - if (credsMap.containsKey(PolarisCredentialProperty.EXPIRATION_TIME)) { - return Long.parseLong(credsMap.get(PolarisCredentialProperty.EXPIRATION_TIME)); + if (credsMap.containsKey(StorageAccessProperty.EXPIRATION_TIME)) { + return Long.parseLong(credsMap.get(StorageAccessProperty.EXPIRATION_TIME)); } return Long.MAX_VALUE; } @@ -57,17 +57,17 @@ public long getExpirationTime() { * account endpoint */ private void handleAzureCredential( - HashMap results, PolarisCredentialProperty credentialProperty, String value) { - if (credentialProperty.equals(PolarisCredentialProperty.AZURE_SAS_TOKEN)) { - String host = credsMap.get(PolarisCredentialProperty.AZURE_ACCOUNT_HOST); - results.put(credentialProperty.getPropertyName() + host, value); + BiConsumer results, StorageAccessProperty credentialProperty, String value) { + if (credentialProperty.equals(StorageAccessProperty.AZURE_SAS_TOKEN)) { + String host = credsMap.get(StorageAccessProperty.AZURE_ACCOUNT_HOST); + results.accept(credentialProperty.getPropertyName() + host, value); // Iceberg 1.7.x may expect the credential key to _not_ be suffixed with endpoint if (host.endsWith(AzureLocation.ADLS_ENDPOINT)) { int suffixIndex = host.lastIndexOf(AzureLocation.ADLS_ENDPOINT) - 1; if (suffixIndex > 0) { String withSuffixStripped = host.substring(0, suffixIndex); - results.put(credentialProperty.getPropertyName() + withSuffixStripped, value); + results.accept(credentialProperty.getPropertyName() + withSuffixStripped, value); } } @@ -75,7 +75,7 @@ private void handleAzureCredential( int suffixIndex = host.lastIndexOf(AzureLocation.BLOB_ENDPOINT) - 1; if (suffixIndex > 0) { String withSuffixStripped = host.substring(0, suffixIndex); - results.put(credentialProperty.getPropertyName() + withSuffixStripped, value); + results.accept(credentialProperty.getPropertyName() + withSuffixStripped, value); } } } @@ -86,18 +86,23 @@ private void handleAzureCredential( * * @return a map of string representing the subscoped creds info. */ - public Map convertToMapOfString() { - HashMap resCredsMap = new HashMap<>(); + AccessConfig toAccessConfig() { + ImmutableAccessConfig.Builder config = AccessConfig.builder(); if (!credsMap.isEmpty()) { credsMap.forEach( (key, value) -> { - if (key.equals(PolarisCredentialProperty.AZURE_SAS_TOKEN)) { - handleAzureCredential(resCredsMap, key, value); - } else if (!key.equals(PolarisCredentialProperty.AZURE_ACCOUNT_HOST)) { - resCredsMap.put(key.getPropertyName(), value); + if (!key.isCredential()) { + config.putExtraProperty(key.getPropertyName(), value); + return; + } + + if (key.equals(StorageAccessProperty.AZURE_SAS_TOKEN)) { + handleAzureCredential(config::putCredential, key, value); + } else if (!key.equals(StorageAccessProperty.AZURE_ACCOUNT_HOST)) { + config.putCredential(key.getPropertyName(), value); } }); } - return resCredsMap; + return config.build(); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java index 7d886e32c6..3a7c85780d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java @@ -40,8 +40,8 @@ import java.util.stream.Stream; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.storage.InMemoryStorageIntegration; -import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.StorageUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,7 +69,7 @@ public GcpCredentialsStorageIntegration( } @Override - public EnumMap getSubscopedCreds( + public EnumMap getSubscopedCreds( @Nonnull PolarisDiagnostics diagnostics, @Nonnull GcpStorageConfigurationInfo storageConfig, boolean allowListOperation, @@ -106,11 +106,10 @@ public EnumMap getSubscopedCreds( // If expires_in missing, use source credential's expire time, which require another api call to // get. - EnumMap propertyMap = - new EnumMap<>(PolarisCredentialProperty.class); - propertyMap.put(PolarisCredentialProperty.GCS_ACCESS_TOKEN, token.getTokenValue()); + EnumMap propertyMap = new EnumMap<>(StorageAccessProperty.class); + propertyMap.put(StorageAccessProperty.GCS_ACCESS_TOKEN, token.getTokenValue()); propertyMap.put( - PolarisCredentialProperty.GCS_ACCESS_TOKEN_EXPIRES_AT, + StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT, String.valueOf(token.getExpirationTime().getTime())); return propertyMap; } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java index 0f1f4f151b..12e83e2d2f 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java @@ -207,7 +207,7 @@ public MockInMemoryStorageIntegration() { } @Override - public EnumMap getSubscopedCreds( + public EnumMap getSubscopedCreds( @Nonnull PolarisDiagnostics diagnostics, @Nonnull PolarisStorageConfigurationInfo storageConfig, boolean allowListOperation, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index 618ddddb56..8856ddfea6 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -44,7 +44,8 @@ import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.persistence.transactional.TreeMapMetaStore; import org.apache.polaris.core.persistence.transactional.TreeMapTransactionalPersistenceImpl; -import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.AccessConfig; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.azure.AzureLocation; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.RepeatedTest; @@ -395,28 +396,28 @@ private static List getFakeScopedCreds(int number, bool res.add( new ScopedCredentialsResult( new EnumMap<>( - ImmutableMap.builder() - .put(PolarisCredentialProperty.AWS_KEY_ID, "key_id_" + finalI) - .put(PolarisCredentialProperty.AWS_SECRET_KEY, "key_secret_" + finalI) - .put(PolarisCredentialProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, expireTime) - .put(PolarisCredentialProperty.EXPIRATION_TIME, expireTime) + ImmutableMap.builder() + .put(StorageAccessProperty.AWS_KEY_ID, "key_id_" + finalI) + .put(StorageAccessProperty.AWS_SECRET_KEY, "key_secret_" + finalI) + .put(StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, expireTime) + .put(StorageAccessProperty.EXPIRATION_TIME, expireTime) .buildOrThrow()))); if (res.size() == number) return res; res.add( new ScopedCredentialsResult( new EnumMap<>( - ImmutableMap.builder() - .put(PolarisCredentialProperty.AZURE_SAS_TOKEN, "sas_token_" + finalI) - .put(PolarisCredentialProperty.AZURE_ACCOUNT_HOST, "account_host") - .put(PolarisCredentialProperty.EXPIRATION_TIME, expireTime) + ImmutableMap.builder() + .put(StorageAccessProperty.AZURE_SAS_TOKEN, "sas_token_" + finalI) + .put(StorageAccessProperty.AZURE_ACCOUNT_HOST, "account_host") + .put(StorageAccessProperty.EXPIRATION_TIME, expireTime) .buildOrThrow()))); if (res.size() == number) return res; res.add( new ScopedCredentialsResult( new EnumMap<>( - ImmutableMap.builder() - .put(PolarisCredentialProperty.GCS_ACCESS_TOKEN, "gcs_token_" + finalI) - .put(PolarisCredentialProperty.GCS_ACCESS_TOKEN_EXPIRES_AT, expireTime) + ImmutableMap.builder() + .put(StorageAccessProperty.GCS_ACCESS_TOKEN, "gcs_token_" + finalI) + .put(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT, expireTime) .buildOrThrow()))); } return res; @@ -447,34 +448,28 @@ public void testAzureCredentialFormatting() { List.of( new ScopedCredentialsResult( new EnumMap<>( - ImmutableMap.builder() - .put(PolarisCredentialProperty.AZURE_SAS_TOKEN, "sas_token_azure_1") - .put(PolarisCredentialProperty.AZURE_ACCOUNT_HOST, "some_account") - .put( - PolarisCredentialProperty.EXPIRATION_TIME, - String.valueOf(Long.MAX_VALUE)) + ImmutableMap.builder() + .put(StorageAccessProperty.AZURE_SAS_TOKEN, "sas_token_azure_1") + .put(StorageAccessProperty.AZURE_ACCOUNT_HOST, "some_account") + .put(StorageAccessProperty.EXPIRATION_TIME, String.valueOf(Long.MAX_VALUE)) .buildOrThrow())), new ScopedCredentialsResult( new EnumMap<>( - ImmutableMap.builder() - .put(PolarisCredentialProperty.AZURE_SAS_TOKEN, "sas_token_azure_2") + ImmutableMap.builder() + .put(StorageAccessProperty.AZURE_SAS_TOKEN, "sas_token_azure_2") .put( - PolarisCredentialProperty.AZURE_ACCOUNT_HOST, + StorageAccessProperty.AZURE_ACCOUNT_HOST, "some_account." + AzureLocation.ADLS_ENDPOINT) - .put( - PolarisCredentialProperty.EXPIRATION_TIME, - String.valueOf(Long.MAX_VALUE)) + .put(StorageAccessProperty.EXPIRATION_TIME, String.valueOf(Long.MAX_VALUE)) .buildOrThrow())), new ScopedCredentialsResult( new EnumMap<>( - ImmutableMap.builder() - .put(PolarisCredentialProperty.AZURE_SAS_TOKEN, "sas_token_azure_3") + ImmutableMap.builder() + .put(StorageAccessProperty.AZURE_SAS_TOKEN, "sas_token_azure_3") .put( - PolarisCredentialProperty.AZURE_ACCOUNT_HOST, + StorageAccessProperty.AZURE_ACCOUNT_HOST, "some_account." + AzureLocation.BLOB_ENDPOINT) - .put( - PolarisCredentialProperty.EXPIRATION_TIME, - String.valueOf(Long.MAX_VALUE)) + .put(StorageAccessProperty.EXPIRATION_TIME, String.valueOf(Long.MAX_VALUE)) .buildOrThrow()))); Mockito.when( @@ -492,40 +487,84 @@ public void testAzureCredentialFormatting() { List entityList = getPolarisEntities(); Map noSuffixResult = - storageCredentialCache.getOrGenerateSubScopeCreds( - metaStoreManager, - callCtx, - entityList.get(0), - true, - new HashSet<>(Arrays.asList("s3://bucket1/path", "s3://bucket2/path")), - new HashSet<>(Arrays.asList("s3://bucket3/path", "s3://bucket4/path"))); + storageCredentialCache + .getOrGenerateSubScopeCreds( + metaStoreManager, + callCtx, + entityList.get(0), + true, + new HashSet<>(Arrays.asList("s3://bucket1/path", "s3://bucket2/path")), + new HashSet<>(Arrays.asList("s3://bucket3/path", "s3://bucket4/path"))) + .credentials(); Assertions.assertThat(noSuffixResult.size()).isEqualTo(2); Assertions.assertThat(noSuffixResult).containsKey("adls.sas-token.some_account"); Map adlsSuffixResult = - storageCredentialCache.getOrGenerateSubScopeCreds( - metaStoreManager, - callCtx, - entityList.get(1), - true, - new HashSet<>(Arrays.asList("s3://bucket1/path", "s3://bucket2/path")), - new HashSet<>(Arrays.asList("s3://bucket3/path", "s3://bucket4/path"))); + storageCredentialCache + .getOrGenerateSubScopeCreds( + metaStoreManager, + callCtx, + entityList.get(1), + true, + new HashSet<>(Arrays.asList("s3://bucket1/path", "s3://bucket2/path")), + new HashSet<>(Arrays.asList("s3://bucket3/path", "s3://bucket4/path"))) + .credentials(); Assertions.assertThat(adlsSuffixResult.size()).isEqualTo(3); Assertions.assertThat(adlsSuffixResult).containsKey("adls.sas-token.some_account"); Assertions.assertThat(adlsSuffixResult) .containsKey("adls.sas-token.some_account." + AzureLocation.ADLS_ENDPOINT); Map blobSuffixResult = + storageCredentialCache + .getOrGenerateSubScopeCreds( + metaStoreManager, + callCtx, + entityList.get(2), + true, + new HashSet<>(Arrays.asList("s3://bucket1/path", "s3://bucket2/path")), + new HashSet<>(Arrays.asList("s3://bucket3/path", "s3://bucket4/path"))) + .credentials(); + Assertions.assertThat(blobSuffixResult.size()).isEqualTo(3); + Assertions.assertThat(blobSuffixResult).containsKey("adls.sas-token.some_account"); + Assertions.assertThat(blobSuffixResult) + .containsKey("adls.sas-token.some_account." + AzureLocation.BLOB_ENDPOINT); + } + + @Test + public void testExtraProperties() { + storageCredentialCache = new StorageCredentialCache(); + ScopedCredentialsResult properties = + new ScopedCredentialsResult( + new EnumMap<>( + ImmutableMap.builder() + .put(StorageAccessProperty.AWS_SECRET_KEY, "super-secret-123") + .put(StorageAccessProperty.AWS_ENDPOINT, "test-endpoint1") + .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS, "true") + .buildOrThrow())); + Mockito.when( + metaStoreManager.getSubscopedCredsForEntity( + Mockito.any(), + Mockito.anyLong(), + Mockito.anyLong(), + Mockito.any(), + Mockito.anyBoolean(), + Mockito.anySet(), + Mockito.anySet())) + .thenReturn(properties); + List entityList = getPolarisEntities(); + + AccessConfig config = storageCredentialCache.getOrGenerateSubScopeCreds( metaStoreManager, callCtx, - entityList.get(2), + entityList.get(0), true, new HashSet<>(Arrays.asList("s3://bucket1/path", "s3://bucket2/path")), new HashSet<>(Arrays.asList("s3://bucket3/path", "s3://bucket4/path"))); - Assertions.assertThat(blobSuffixResult.size()).isEqualTo(3); - Assertions.assertThat(blobSuffixResult).containsKey("adls.sas-token.some_account"); - Assertions.assertThat(blobSuffixResult) - .containsKey("adls.sas-token.some_account." + AzureLocation.BLOB_ENDPOINT); + Assertions.assertThat(config.credentials()) + .containsExactly(Map.entry("s3.secret-access-key", "super-secret-123")); + Assertions.assertThat(config.extraProperties()) + .containsExactlyInAnyOrderEntriesOf( + Map.of("s3.endpoint", "test-endpoint1", "s3.path-style-access", "true")); } } diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java index 9a68867539..b837033e17 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/aws/AwsCredentialsStorageIntegrationTest.java @@ -26,8 +26,8 @@ import java.util.List; import java.util.Set; import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.assertj.core.api.Assertions; @@ -80,7 +80,7 @@ public void testGetSubscopedCreds() { return ASSUME_ROLE_RESPONSE; }); String warehouseDir = "s3://bucket/path/to/warehouse"; - EnumMap credentials = + EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( Mockito.mock(PolarisDiagnostics.class), @@ -95,11 +95,11 @@ public void testGetSubscopedCreds() { Set.of(warehouseDir + "/namespace/table")); assertThat(credentials) .isNotEmpty() - .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") - .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") - .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey") + .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess") + .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey") + .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey") .containsEntry( - PolarisCredentialProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, + StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, String.valueOf(EXPIRE_TIME.toEpochMilli())); } @@ -245,7 +245,7 @@ public void testGetSubscopedCredsInlinePolicy(String awsPartition) { break; case AWS_PARTITION: case "aws-us-gov": - EnumMap credentials = + EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( Mockito.mock(PolarisDiagnostics.class), @@ -260,11 +260,11 @@ public void testGetSubscopedCredsInlinePolicy(String awsPartition) { Set.of(s3Path(bucket, firstPath))); assertThat(credentials) .isNotEmpty() - .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") - .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") - .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey") + .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess") + .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey") + .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey") .containsEntry( - PolarisCredentialProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, + StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, String.valueOf(EXPIRE_TIME.toEpochMilli())); break; default: @@ -346,7 +346,7 @@ public void testGetSubscopedCredsInlinePolicyWithoutList() { }); PolarisStorageConfigurationInfo.StorageType storageType = PolarisStorageConfigurationInfo.StorageType.S3; - EnumMap credentials = + EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( Mockito.mock(PolarisDiagnostics.class), @@ -361,11 +361,11 @@ public void testGetSubscopedCredsInlinePolicyWithoutList() { Set.of(s3Path(bucket, firstPath))); assertThat(credentials) .isNotEmpty() - .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") - .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") - .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey") + .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess") + .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey") + .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey") .containsEntry( - PolarisCredentialProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, + StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, String.valueOf(EXPIRE_TIME.toEpochMilli())); } @@ -441,7 +441,7 @@ public void testGetSubscopedCredsInlinePolicyWithoutWrites() { }); PolarisStorageConfigurationInfo.StorageType storageType = PolarisStorageConfigurationInfo.StorageType.S3; - EnumMap credentials = + EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( Mockito.mock(PolarisDiagnostics.class), @@ -456,11 +456,11 @@ public void testGetSubscopedCredsInlinePolicyWithoutWrites() { Set.of()); assertThat(credentials) .isNotEmpty() - .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") - .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") - .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey") + .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess") + .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey") + .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey") .containsEntry( - PolarisCredentialProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, + StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, String.valueOf(EXPIRE_TIME.toEpochMilli())); } @@ -506,7 +506,7 @@ public void testGetSubscopedCredsInlinePolicyWithEmptyReadAndWrite() { }); return ASSUME_ROLE_RESPONSE; }); - EnumMap credentials = + EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( Mockito.mock(PolarisDiagnostics.class), @@ -521,11 +521,11 @@ public void testGetSubscopedCredsInlinePolicyWithEmptyReadAndWrite() { Set.of()); assertThat(credentials) .isNotEmpty() - .containsEntry(PolarisCredentialProperty.AWS_TOKEN, "sess") - .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, "accessKey") - .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, "secretKey") + .containsEntry(StorageAccessProperty.AWS_TOKEN, "sess") + .containsEntry(StorageAccessProperty.AWS_KEY_ID, "accessKey") + .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, "secretKey") .containsEntry( - PolarisCredentialProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, + StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, String.valueOf(EXPIRE_TIME.toEpochMilli())); } @@ -563,7 +563,7 @@ public void testClientRegion(String awsPartition) { break; case AWS_PARTITION: case "aws-us-gov": - EnumMap credentials = + EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( Mockito.mock(PolarisDiagnostics.class), @@ -578,7 +578,7 @@ public void testClientRegion(String awsPartition) { Set.of()); assertThat(credentials) .isNotEmpty() - .containsEntry(PolarisCredentialProperty.CLIENT_REGION, clientRegion); + .containsEntry(StorageAccessProperty.CLIENT_REGION, clientRegion); break; default: throw new IllegalArgumentException("Unknown aws partition: " + awsPartition); @@ -601,7 +601,7 @@ public void testNoClientRegion(String awsPartition) { }); switch (awsPartition) { case AWS_PARTITION: - EnumMap credentials = + EnumMap credentials = new AwsCredentialsStorageIntegration(stsClient) .getSubscopedCreds( Mockito.mock(PolarisDiagnostics.class), @@ -614,9 +614,7 @@ public void testNoClientRegion(String awsPartition) { true, /* allowList = true */ Set.of(), Set.of()); - assertThat(credentials) - .isNotEmpty() - .doesNotContainKey(PolarisCredentialProperty.CLIENT_REGION); + assertThat(credentials).isNotEmpty().doesNotContainKey(StorageAccessProperty.CLIENT_REGION); break; case "aws-cn": case "aws-us-gov": diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java index fd7ac9a880..f4738d73e1 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/azure/AzureCredentialStorageIntegrationTest.java @@ -48,7 +48,7 @@ import java.util.Map; import java.util.stream.Stream; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration; import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; import org.assertj.core.api.Assertions; @@ -120,13 +120,13 @@ public void testGetSubscopedTokenList(boolean allowListAction, String service) { String.format( "abfss://container@icebergdfsstorageacct.%s.core.windows.net/polaris-test/", service)); - Map credsMap = + Map credsMap = subscopedCredsForOperations( /* allowedReadLoc= */ allowedLoc, /* allowedWriteLoc= */ new ArrayList<>(), allowListAction); Assertions.assertThat(credsMap).hasSize(2); - String sasToken = credsMap.get(PolarisCredentialProperty.AZURE_SAS_TOKEN); + String sasToken = credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN); Assertions.assertThat(sasToken).isNotNull(); String serviceEndpoint = String.format("https://icebergdfsstorageacct.%s.core.windows.net", service); @@ -191,7 +191,7 @@ public void testGetSubscopedTokenRead( String.format( "abfss://container@icebergdfsstorageacct.%s.core.windows.net/%s", service, allowedPrefix)); - Map credsMap = + Map credsMap = subscopedCredsForOperations( /* allowedReadLoc= */ allowedLoc, /* allowedWriteLoc= */ new ArrayList<>(), @@ -199,7 +199,7 @@ public void testGetSubscopedTokenRead( BlobClient blobClient = createBlobClient( - credsMap.get(PolarisCredentialProperty.AZURE_SAS_TOKEN), + credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN), "https://icebergdfsstorageacct.dfs.core.windows.net", "container", allowedPrefix); @@ -230,7 +230,7 @@ public void testGetSubscopedTokenRead( // read fail because container is blocked BlobClient blobClientReadFail = createBlobClient( - credsMap.get(PolarisCredentialProperty.AZURE_SAS_TOKEN), + credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN), String.format("https://icebergdfsstorageacct.%s.core.windows.net", service), "regtest", blockedPrefix); @@ -261,7 +261,7 @@ public void testGetSubscopedTokenWrite( String.format( "abfss://container@icebergdfsstorageacct.%s.core.windows.net/%s", service, allowedPrefix)); - Map credsMap = + Map credsMap = subscopedCredsForOperations( /* allowedReadLoc= */ new ArrayList<>(), /* allowedWriteLoc= */ allowedLoc, @@ -270,13 +270,13 @@ public void testGetSubscopedTokenWrite( String.format("https://icebergdfsstorageacct.%s.core.windows.net", service); BlobClient blobClient = createBlobClient( - credsMap.get(PolarisCredentialProperty.AZURE_SAS_TOKEN), + credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN), serviceEndpoint, "container", allowedPrefix + "metadata/00000-65ffa17b-fe64-4c38-bcb9-06f9bd12aa2a.metadata.json"); DataLakeFileClient fileClient = createDatalakeFileClient( - credsMap.get(PolarisCredentialProperty.AZURE_SAS_TOKEN), + credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN), serviceEndpoint, "container", "polaris-test/scopedcreds/metadata", @@ -311,13 +311,13 @@ public void testGetSubscopedTokenWrite( String blockedContainer = "regtest"; BlobClient blobClientWriteFail = createBlobClient( - credsMap.get(PolarisCredentialProperty.AZURE_SAS_TOKEN), + credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN), serviceEndpoint, blockedContainer, blockedPrefix); DataLakeFileClient fileClientFail = createDatalakeFileClient( - credsMap.get(PolarisCredentialProperty.AZURE_SAS_TOKEN), + credsMap.get(StorageAccessProperty.AZURE_SAS_TOKEN), serviceEndpoint, blockedContainer, "polaris-test/scopedcreds/metadata", @@ -338,7 +338,7 @@ public void testGetSubscopedTokenWrite( } } - private Map subscopedCredsForOperations( + private Map subscopedCredsForOperations( List allowedReadLoc, List allowedWriteLoc, boolean allowListAction) { List allowedLoc = new ArrayList<>(); allowedLoc.addAll(allowedReadLoc); @@ -347,7 +347,7 @@ private Map subscopedCredsForOperations( new AzureStorageConfigurationInfo(allowedLoc, tenantId); AzureCredentialsStorageIntegration azureCredsIntegration = new AzureCredentialsStorageIntegration(); - EnumMap credsMap = + EnumMap credsMap = azureCredsIntegration.getSubscopedCreds( new PolarisDefaultDiagServiceImpl(), azureConfig, diff --git a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java index 35d05c2513..fc88bd8170 100644 --- a/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/service/storage/gcp/GcpCredentialsStorageIntegrationTest.java @@ -49,7 +49,7 @@ import java.util.Map; import java.util.Set; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; -import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo; import org.assertj.core.api.Assertions; @@ -136,7 +136,7 @@ public void testSubscope(boolean allowedListAction) throws Exception { private Storage setupStorageClient( List allowedReadLoc, List allowedWriteLoc, boolean allowListAction) throws IOException { - Map credsMap = + Map credsMap = subscopedCredsForOperations(allowedReadLoc, allowedWriteLoc, allowListAction); return createStorageClient(credsMap); } @@ -146,20 +146,19 @@ BlobInfo createStorageBlob(String bucket, String prefix, String fileName) { return BlobInfo.newBuilder(blobId).build(); } - private Storage createStorageClient(Map credsMap) { + private Storage createStorageClient(Map credsMap) { AccessToken accessToken = new AccessToken( - credsMap.get(PolarisCredentialProperty.GCS_ACCESS_TOKEN), + credsMap.get(StorageAccessProperty.GCS_ACCESS_TOKEN), new Date( - Long.parseLong( - credsMap.get(PolarisCredentialProperty.GCS_ACCESS_TOKEN_EXPIRES_AT)))); + Long.parseLong(credsMap.get(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT)))); return StorageOptions.newBuilder() .setCredentials(GoogleCredentials.create(accessToken)) .build() .getService(); } - private Map subscopedCredsForOperations( + private Map subscopedCredsForOperations( List allowedReadLoc, List allowedWriteLoc, boolean allowListAction) throws IOException { List allowedLoc = new ArrayList<>(); @@ -170,7 +169,7 @@ private Map subscopedCredsForOperations( new GcpCredentialsStorageIntegration( GoogleCredentials.getApplicationDefault(), ServiceOptions.getFromServiceLoader(HttpTransportFactory.class, NetHttpTransport::new)); - EnumMap credsMap = + EnumMap credsMap = gcpCredsIntegration.getSubscopedCreds( new PolarisDefaultDiagServiceImpl(), gcpConfig, diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 5a83330a5b..fbafd37c9a 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -101,10 +101,10 @@ import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.secrets.UserSecretsManagerFactory; -import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; @@ -1520,7 +1520,7 @@ public void testDropTableWithPurge() { metaStoreManager.loadTasks(polarisContext, "testExecutor", 1).getEntities(); Assertions.assertThat(tasks).hasSize(1); TaskEntity taskEntity = TaskEntity.of(tasks.get(0)); - EnumMap credentials = + EnumMap credentials = metaStoreManager .getSubscopedCredsForEntity( polarisContext, @@ -1534,9 +1534,9 @@ public void testDropTableWithPurge() { Assertions.assertThat(credentials) .isNotNull() .isNotEmpty() - .containsEntry(PolarisCredentialProperty.AWS_KEY_ID, TEST_ACCESS_KEY) - .containsEntry(PolarisCredentialProperty.AWS_SECRET_KEY, SECRET_ACCESS_KEY) - .containsEntry(PolarisCredentialProperty.AWS_TOKEN, SESSION_TOKEN); + .containsEntry(StorageAccessProperty.AWS_KEY_ID, TEST_ACCESS_KEY) + .containsEntry(StorageAccessProperty.AWS_SECRET_KEY, SECRET_ACCESS_KEY) + .containsEntry(StorageAccessProperty.AWS_TOKEN, SESSION_TOKEN); MetaStoreManagerFactory metaStoreManagerFactory = createMockMetaStoreManagerFactory(); FileIO fileIO = new TaskFileIOSupplier( diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 27ef12c3d7..afa7d14773 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -113,6 +113,7 @@ import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.storage.AccessConfig; import org.apache.polaris.core.storage.InMemoryStorageIntegration; import org.apache.polaris.core.storage.PolarisCredentialVendor; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -863,7 +864,7 @@ public boolean sendNotification( } @Override - public Map getCredentialConfig( + public AccessConfig getAccessConfig( TableIdentifier tableIdentifier, TableMetadata tableMetadata, Set storageActions) { @@ -873,9 +874,9 @@ public Map getCredentialConfig( .atWarn() .addKeyValue("tableIdentifier", tableIdentifier) .log("Table entity has no storage configuration in its hierarchy"); - return Map.of(); + return AccessConfig.builder().build(); } - return FileIOUtil.refreshCredentials( + return FileIOUtil.refreshAccessConfig( callContext, entityManager, getCredentialVendor(), diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 2785e004cf..dbea9f4d74 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -91,6 +91,7 @@ import org.apache.polaris.core.persistence.dao.entity.EntitiesResult; import org.apache.polaris.core.persistence.dao.entity.EntityWithPath; import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.storage.AccessConfig; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.service.catalog.SupportsNotifications; import org.apache.polaris.service.catalog.common.CatalogHandler; @@ -709,9 +710,11 @@ private LoadTableResponse.Builder buildLoadTableResponseWithDelegationCredential .addKeyValue("tableIdentifier", tableIdentifier) .addKeyValue("tableLocation", tableMetadata.location()) .log("Fetching client credentials for table"); - Map credentialConfig = - credentialDelegation.getCredentialConfig(tableIdentifier, tableMetadata, actions); + AccessConfig accessConfig = + credentialDelegation.getAccessConfig(tableIdentifier, tableMetadata, actions); + Map credentialConfig = accessConfig.credentials(); responseBuilder.addAllConfig(credentialConfig); + responseBuilder.addAllConfig(accessConfig.extraProperties()); if (!credentialConfig.isEmpty()) { responseBuilder.addCredential( ImmutableCredential.builder() diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java index 06ca7fbde6..21ec380eb0 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java @@ -18,10 +18,10 @@ */ package org.apache.polaris.service.catalog.iceberg; -import java.util.Map; import java.util.Set; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.storage.AccessConfig; import org.apache.polaris.core.storage.PolarisStorageActions; /** @@ -32,7 +32,7 @@ * configuration. */ public interface SupportsCredentialDelegation { - Map getCredentialConfig( + AccessConfig getAccessConfig( TableIdentifier tableIdentifier, TableMetadata tableMetadata, Set storageActions); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index 31f2f9b03a..db4874521a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -38,6 +38,7 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.storage.AccessConfig; import org.apache.polaris.core.storage.PolarisCredentialVendor; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.service.config.RealmEntityManagerFactory; @@ -87,26 +88,27 @@ public FileIO loadFileIO( properties = new HashMap<>(properties); Optional storageInfoEntity = FileIOUtil.findStorageInfoFromHierarchy(resolvedEntityPath); - Map credentialsMap = - storageInfoEntity - .map( - storageInfo -> - FileIOUtil.refreshCredentials( - callContext, - entityManager, - credentialVendor, - configurationStore, - identifier, - tableLocations, - storageActions, - storageInfo)) - .orElse(Map.of()); + Optional accessConfig = + storageInfoEntity.map( + storageInfo -> + FileIOUtil.refreshAccessConfig( + callContext, + entityManager, + credentialVendor, + configurationStore, + identifier, + tableLocations, + storageActions, + storageInfo)); // Update the FileIO with the subscoped credentials // Update with properties in case there are table-level overrides the credentials should // always override table-level properties, since storage configuration will be found at // whatever entity defines it - properties.putAll(credentialsMap); + if (accessConfig.isPresent()) { + properties.putAll(accessConfig.get().credentials()); + properties.putAll(accessConfig.get().extraProperties()); + } return loadFileIOInternal(ioImplClassName, properties); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java index 481033dad0..e6c02ee542 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java @@ -18,7 +18,6 @@ */ package org.apache.polaris.service.catalog.io; -import java.util.Map; import java.util.Optional; import java.util.Set; import org.apache.iceberg.catalog.TableIdentifier; @@ -29,6 +28,7 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.storage.AccessConfig; import org.apache.polaris.core.storage.PolarisCredentialVendor; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; @@ -75,7 +75,7 @@ public static Optional findStorageInfoFromHierarchy( * and read/write metadata JSON files. * */ - public static Map refreshCredentials( + public static AccessConfig refreshAccessConfig( CallContext callContext, PolarisEntityManager entityManager, PolarisCredentialVendor credentialVendor, @@ -95,7 +95,7 @@ public static Map refreshCredentials( .atDebug() .addKeyValue("tableIdentifier", tableIdentifier) .log("Skipping generation of subscoped creds for table"); - return Map.of(); + return AccessConfig.builder().build(); } boolean allowList = @@ -107,7 +107,7 @@ public static Map refreshCredentials( || storageActions.contains(PolarisStorageActions.ALL) ? tableLocations : Set.of(); - Map credentialsMap = + AccessConfig accessConfig = entityManager .getCredentialCache() .getOrGenerateSubScopeCreds( @@ -120,11 +120,12 @@ public static Map refreshCredentials( LOGGER .atDebug() .addKeyValue("tableIdentifier", tableIdentifier) - .addKeyValue("credentialKeys", credentialsMap.keySet()) + .addKeyValue("credentialKeys", accessConfig.credentials().keySet()) + .addKeyValue("extraProperties", accessConfig.extraProperties()) .log("Loaded scoped credentials for table"); - if (credentialsMap.isEmpty()) { + if (accessConfig.credentials().isEmpty()) { LOGGER.debug("No credentials found for table"); } - return credentialsMap; + return accessConfig; } } diff --git a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index f61c67620f..0c129ce478 100644 --- a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -31,11 +31,11 @@ import java.util.Set; import java.util.function.Supplier; import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.core.storage.StorageAccessProperty; import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; import org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration; import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; @@ -89,13 +89,13 @@ public PolarisStorageIntegrationProviderImpl( storageIntegration = new PolarisStorageIntegration<>("file") { @Override - public EnumMap getSubscopedCreds( + public EnumMap getSubscopedCreds( @Nonnull PolarisDiagnostics diagnostics, @Nonnull T storageConfig, boolean allowListOperation, @Nonnull Set allowedReadLocations, @Nonnull Set allowedWriteLocations) { - return new EnumMap<>(PolarisCredentialProperty.class); + return new EnumMap<>(StorageAccessProperty.class); } @Override