From d75703c6e3c44129bee9e00c6f1263c54e8c3f86 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Tue, 27 May 2025 16:52:55 -0700 Subject: [PATCH 01/15] Add SigV4 auth related DPOs --- .../AuthenticationParametersDpo.java | 23 +++- .../core/connection/AuthenticationType.java | 3 +- .../BearerAuthenticationParametersDpo.java | 5 +- .../connection/ConnectionConfigInfoDpo.java | 31 ++++- .../OAuthClientCredentialsParametersDpo.java | 7 +- .../SigV4AuthenticationParametersDpo.java | 129 ++++++++++++++++++ .../hadoop/HadoopConnectionConfigInfoDpo.java | 24 +++- .../IcebergCatalogPropertiesProvider.java | 4 +- .../IcebergRestConnectionConfigInfoDpo.java | 28 +++- .../core/identity/ServiceIdentityType.java | 83 +++++++++++ .../dpo/AwsIamServiceIdentityInfoDpo.java | 100 ++++++++++++++ .../identity/dpo/ServiceIdentityInfoDpo.java | 79 +++++++++++ .../polaris/core/secrets/SecretReference.java | 103 ++++++++++++++ .../core/secrets/ServiceSecretReference.java | 60 ++++++++ .../core/secrets/UserSecretReference.java | 59 +------- 15 files changed, 663 insertions(+), 75 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityType.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/AwsIamServiceIdentityInfoDpo.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/ServiceIdentityInfoDpo.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/secrets/SecretReference.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/secrets/ServiceSecretReference.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java index f2267b12a5..20d1e88a34 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java @@ -18,13 +18,16 @@ */ package org.apache.polaris.core.connection; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.annotation.Nonnull; import java.util.Map; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; +import org.apache.polaris.core.admin.model.SigV4AuthenticationParameters; import org.apache.polaris.core.connection.iceberg.IcebergCatalogPropertiesProvider; import org.apache.polaris.core.secrets.UserSecretReference; @@ -39,6 +42,7 @@ @JsonSubTypes({ @JsonSubTypes.Type(value = OAuthClientCredentialsParametersDpo.class, name = "1"), @JsonSubTypes.Type(value = BearerAuthenticationParametersDpo.class, name = "2"), + @JsonSubTypes.Type(value = SigV4AuthenticationParametersDpo.class, name = "3"), }) public abstract class AuthenticationParametersDpo implements IcebergCatalogPropertiesProvider { @@ -57,7 +61,12 @@ public int getAuthenticationTypeCode() { return authenticationTypeCode; } - public abstract AuthenticationParameters asAuthenticationParametersModel(); + @JsonIgnore + public AuthenticationType getAuthenticationType() { + return AuthenticationType.fromCode(authenticationTypeCode); + } + + public abstract @Nonnull AuthenticationParameters asAuthenticationParametersModel(); public static AuthenticationParametersDpo fromAuthenticationParametersModelWithSecrets( AuthenticationParameters authenticationParameters, @@ -81,6 +90,18 @@ public static AuthenticationParametersDpo fromAuthenticationParametersModelWithS new BearerAuthenticationParametersDpo( secretReferences.get(INLINE_BEARER_TOKEN_REFERENCE_KEY)); break; + case SIGV4: + // SigV4 authentication is not secret-based + SigV4AuthenticationParameters sigV4AuthenticationParametersModel = + (SigV4AuthenticationParameters) authenticationParameters; + config = + new SigV4AuthenticationParametersDpo( + sigV4AuthenticationParametersModel.getRoleArn(), + sigV4AuthenticationParametersModel.getRoleSessionName(), + sigV4AuthenticationParametersModel.getExternalId(), + sigV4AuthenticationParametersModel.getSigningRegion(), + sigV4AuthenticationParametersModel.getSigningName()); + break; default: throw new IllegalStateException( "Unsupported authentication type: " + authenticationParameters.getAuthenticationType()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java index 71ad0b72d3..2eeaae7209 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java @@ -33,6 +33,7 @@ public enum AuthenticationType { NULL_TYPE(0), OAUTH(1), BEARER(2), + SIGV4(3), ; private static final AuthenticationType[] REVERSE_MAPPING_ARRAY; @@ -65,7 +66,7 @@ public enum AuthenticationType { * NULL_TYPE if not found * * @param authTypeCode code associated to the authentication type - * @return ConnectionType corresponding to that code or null if mapping not found + * @return AuthenticationType corresponding to that code or null if mapping not found */ public static @Nonnull AuthenticationType fromCode(int authTypeCode) { // ensure it is within bounds diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java index bf80c7c4cb..a04d2560c0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java @@ -25,6 +25,7 @@ import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; +import org.apache.polaris.core.credentials.PolarisCredentialManager; import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.secrets.UserSecretsManager; @@ -50,13 +51,13 @@ public BearerAuthenticationParametersDpo( @Override public @Nonnull Map asIcebergCatalogProperties( - UserSecretsManager secretsManager) { + UserSecretsManager secretsManager, PolarisCredentialManager credentialManager) { String bearerToken = secretsManager.readSecret(getBearerTokenReference()); return Map.of(OAuth2Properties.TOKEN, bearerToken); } @Override - public AuthenticationParameters asAuthenticationParametersModel() { + public @Nonnull AuthenticationParameters asAuthenticationParametersModel() { return BearerAuthenticationParameters.builder() .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.BEARER) .build(); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java index 4313ede124..97ffa9ed05 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java @@ -18,10 +18,7 @@ */ package org.apache.polaris.core.connection; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -38,6 +35,7 @@ import org.apache.polaris.core.connection.hadoop.HadoopConnectionConfigInfoDpo; import org.apache.polaris.core.connection.iceberg.IcebergCatalogPropertiesProvider; import org.apache.polaris.core.connection.iceberg.IcebergRestConnectionConfigInfoDpo; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; import org.apache.polaris.core.secrets.UserSecretReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,22 +64,29 @@ public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertie // The authentication parameters for the connection private final AuthenticationParametersDpo authenticationParameters; + // The Polaris service identity info of the connection + private final ServiceIdentityInfoDpo serviceIdentity; + public ConnectionConfigInfoDpo( @JsonProperty(value = "connectionTypeCode", required = true) int connectionTypeCode, @JsonProperty(value = "uri", required = true) @Nonnull String uri, @JsonProperty(value = "authenticationParameters", required = true) @Nonnull - AuthenticationParametersDpo authenticationParameters) { - this(connectionTypeCode, uri, authenticationParameters, true); + AuthenticationParametersDpo authenticationParameters, + @JsonProperty(value = "serviceIdentity", required = false) @Nullable + ServiceIdentityInfoDpo serviceIdentity) { + this(connectionTypeCode, uri, authenticationParameters, serviceIdentity, true); } protected ConnectionConfigInfoDpo( int connectionTypeCode, @Nonnull String uri, @Nonnull AuthenticationParametersDpo authenticationParameters, + @Nullable ServiceIdentityInfoDpo serviceIdentity, boolean validateUri) { this.connectionTypeCode = connectionTypeCode; this.uri = uri; this.authenticationParameters = authenticationParameters; + this.serviceIdentity = serviceIdentity; if (validateUri) { validateUri(uri); } @@ -91,6 +96,11 @@ public int getConnectionTypeCode() { return connectionTypeCode; } + @JsonIgnore + public ConnectionType getConnectionType() { + return ConnectionType.fromCode(connectionTypeCode); + } + public String getUri() { return uri; } @@ -99,6 +109,10 @@ public AuthenticationParametersDpo getAuthenticationParameters() { return authenticationParameters; } + public @Nullable ServiceIdentityInfoDpo getServiceIdentity() { + return serviceIdentity; + } + private static final ObjectMapper DEFAULT_MAPPER; static { @@ -157,6 +171,7 @@ public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( new IcebergRestConnectionConfigInfoDpo( icebergRestConfigModel.getUri(), authenticationParameters, + null /*Service Identity Info*/, icebergRestConfigModel.getRemoteCatalogName()); break; case HADOOP: @@ -169,6 +184,7 @@ public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( new HadoopConnectionConfigInfoDpo( hadoopConfigModel.getUri(), authenticationParameters, + null /*Service Identity Info*/, hadoopConfigModel.getWarehouse()); break; default: @@ -178,6 +194,9 @@ public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( return config; } + public abstract ConnectionConfigInfoDpo withServiceIdentity( + ServiceIdentityInfoDpo serviceIdentityInfo); + /** * Produces the correponding API-model ConnectionConfigInfo for this persistence object; many * fields are one-to-one direct mappings, but some fields, such as secretReferences, might only be diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java index 9a955de4fd..4a3020bc15 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java @@ -35,6 +35,7 @@ import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; +import org.apache.polaris.core.credentials.PolarisCredentialManager; import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.secrets.UserSecretsManager; @@ -86,7 +87,7 @@ public OAuthClientCredentialsParametersDpo( return clientSecretReference; } - public @Nonnull List getScopes() { + public @Nullable List getScopes() { return scopes; } @@ -104,7 +105,7 @@ public OAuthClientCredentialsParametersDpo( @Override public @Nonnull Map asIcebergCatalogProperties( - UserSecretsManager secretsManager) { + UserSecretsManager secretsManager, PolarisCredentialManager credentialManager) { HashMap properties = new HashMap<>(); if (getTokenUri() != null) { properties.put(OAuth2Properties.OAUTH2_SERVER_URI, getTokenUri()); @@ -115,7 +116,7 @@ public OAuthClientCredentialsParametersDpo( } @Override - public AuthenticationParameters asAuthenticationParametersModel() { + public @Nonnull AuthenticationParameters asAuthenticationParametersModel() { return OAuthClientCredentialsParameters.builder() .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.OAUTH) .setTokenUri(getTokenUri()) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java new file mode 100644 index 0000000000..7250f49606 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableMap; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Map; +import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.rest.auth.AuthProperties; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.SigV4AuthenticationParameters; +import org.apache.polaris.core.credentials.PolarisCredentialManager; +import org.apache.polaris.core.secrets.UserSecretsManager; + +/** + * The internal persistence-object counterpart to SigV4AuthenticationParameters defined in the API + * model. + */ +public class SigV4AuthenticationParametersDpo extends AuthenticationParametersDpo { + + @JsonProperty(value = "roleArn") + private final String roleArn; + + @JsonProperty(value = "roleSessionName") + private final String roleSessionName; + + @JsonProperty(value = "externalId") + private final String externalId; + + @JsonProperty(value = "signingRegion") + private final String signingRegion; + + @JsonProperty(value = "signingName") + private final String signingName; + + public SigV4AuthenticationParametersDpo( + @JsonProperty(value = "roleArn", required = true) String roleArn, + @JsonProperty(value = "roleSessionName", required = false) String roleSessionName, + @JsonProperty(value = "externalId", required = false) String externalId, + @JsonProperty(value = "signingRegion", required = true) String signingRegion, + @JsonProperty(value = "signingName", required = false) String signingName) { + super(AuthenticationType.SIGV4.getCode()); + this.roleArn = roleArn; + this.roleSessionName = roleSessionName; + this.externalId = externalId; + this.signingRegion = signingRegion; + this.signingName = signingName; + } + + public @Nonnull String getRoleArn() { + return roleArn; + } + + public @Nullable String getRoleSessionName() { + return roleSessionName; + } + + public @Nullable String getExternalId() { + return externalId; + } + + public @Nonnull String getSigningRegion() { + return signingRegion; + } + + public @Nullable String getSigningName() { + return signingName; + } + + @Nonnull + @Override + public Map asIcebergCatalogProperties( + UserSecretsManager secretsManager, PolarisCredentialManager credentialManager) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.put(AuthProperties.AUTH_TYPE, AuthProperties.AUTH_TYPE_SIGV4); + builder.put(AwsProperties.REST_SIGNER_REGION, getSigningRegion()); + if (getSigningName() != null) { + builder.put(AwsProperties.REST_SIGNING_NAME, getSigningName()); + } + // TODO: Add a connection credential provider to get the tmp aws credentials for SigV4 auth + // builder.put(AwsProperties.REST_ACCESS_KEY_ID, "access_key_id"); + // builder.put(AwsProperties.REST_SECRET_ACCESS_KEY, "secret_access_key"); + // builder.put(AwsProperties.REST_SESSION_TOKEN, "session_token"); + return builder.build(); + } + + @Override + public @Nonnull AuthenticationParameters asAuthenticationParametersModel() { + return SigV4AuthenticationParameters.builder() + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.SIGV4) + .setRoleArn(getRoleArn()) + .setRoleSessionName(getRoleSessionName()) + .setExternalId(getExternalId()) + .setSigningRegion(getSigningRegion()) + .setSigningName(getSigningName()) + .build(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("authenticationTypeCode", getAuthenticationTypeCode()) + .add("roleArn", getRoleArn()) + .add("roleSessionName", getRoleSessionName()) + .add("externalId", getExternalId()) + .add("signingRegion", getSigningRegion()) + .add("signingName", getSigningName()) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java index 5f29482c15..90afbf6bf4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java @@ -24,12 +24,15 @@ import jakarta.annotation.Nullable; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import org.apache.iceberg.CatalogProperties; import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.apache.polaris.core.admin.model.HadoopConnectionConfigInfo; import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; import org.apache.polaris.core.connection.ConnectionType; +import org.apache.polaris.core.credentials.PolarisCredentialManager; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; import org.apache.polaris.core.secrets.UserSecretsManager; /** @@ -44,8 +47,10 @@ public HadoopConnectionConfigInfoDpo( @JsonProperty(value = "uri", required = true) @Nonnull String uri, @JsonProperty(value = "authenticationParameters", required = true) @Nonnull AuthenticationParametersDpo authenticationParameters, + @JsonProperty(value = "serviceIdentity", required = false) @Nullable + ServiceIdentityInfoDpo serviceIdentityInfo, @JsonProperty(value = "warehouse", required = false) @Nullable String remoteCatalogName) { - super(ConnectionType.HADOOP.getCode(), uri, authenticationParameters); + super(ConnectionType.HADOOP.getCode(), uri, authenticationParameters, serviceIdentityInfo); this.warehouse = remoteCatalogName; } @@ -65,16 +70,25 @@ public String toString() { @Override public @Nonnull Map asIcebergCatalogProperties( - UserSecretsManager secretsManager) { + UserSecretsManager secretsManager, PolarisCredentialManager credentialManager) { HashMap properties = new HashMap<>(); properties.put(CatalogProperties.URI, getUri()); if (getWarehouse() != null) { properties.put(CatalogProperties.WAREHOUSE_LOCATION, getWarehouse()); } - properties.putAll(getAuthenticationParameters().asIcebergCatalogProperties(secretsManager)); + properties.putAll( + getAuthenticationParameters() + .asIcebergCatalogProperties(secretsManager, credentialManager)); return properties; } + @Override + public ConnectionConfigInfoDpo withServiceIdentity( + @Nonnull ServiceIdentityInfoDpo serviceIdentityInfo) { + return new HadoopConnectionConfigInfoDpo( + getUri(), getAuthenticationParameters(), serviceIdentityInfo, getWarehouse()); + } + @Override public ConnectionConfigInfo asConnectionConfigInfoModel() { return HadoopConnectionConfigInfo.builder() @@ -83,6 +97,10 @@ public ConnectionConfigInfo asConnectionConfigInfoModel() { .setWarehouse(getWarehouse()) .setAuthenticationParameters( getAuthenticationParameters().asAuthenticationParametersModel()) + .setServiceIdentity( + Optional.ofNullable(getServiceIdentity()) + .map(ServiceIdentityInfoDpo::asServiceIdentityInfoModel) + .orElse(null)) .build(); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergCatalogPropertiesProvider.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergCatalogPropertiesProvider.java index 75af01100f..e17218f25e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergCatalogPropertiesProvider.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergCatalogPropertiesProvider.java @@ -20,6 +20,7 @@ import jakarta.annotation.Nonnull; import java.util.Map; +import org.apache.polaris.core.credentials.PolarisCredentialManager; import org.apache.polaris.core.secrets.UserSecretsManager; /** @@ -30,5 +31,6 @@ */ public interface IcebergCatalogPropertiesProvider { @Nonnull - Map asIcebergCatalogProperties(UserSecretsManager secretsManager); + Map asIcebergCatalogProperties( + UserSecretsManager secretsManager, PolarisCredentialManager credentialManager); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java index 236dcee293..ed51e65698 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java @@ -24,12 +24,15 @@ import jakarta.annotation.Nullable; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import org.apache.iceberg.CatalogProperties; import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; import org.apache.polaris.core.connection.ConnectionType; +import org.apache.polaris.core.credentials.PolarisCredentialManager; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; import org.apache.polaris.core.secrets.UserSecretsManager; /** @@ -45,9 +48,12 @@ public IcebergRestConnectionConfigInfoDpo( @JsonProperty(value = "uri", required = true) @Nonnull String uri, @JsonProperty(value = "authenticationParameters", required = true) @Nonnull AuthenticationParametersDpo authenticationParameters, + @JsonProperty(value = "serviceIdentity", required = false) @Nullable + ServiceIdentityInfoDpo serviceIdentityInfo, @JsonProperty(value = "remoteCatalogName", required = false) @Nullable String remoteCatalogName) { - super(ConnectionType.ICEBERG_REST.getCode(), uri, authenticationParameters); + super( + ConnectionType.ICEBERG_REST.getCode(), uri, authenticationParameters, serviceIdentityInfo); this.remoteCatalogName = remoteCatalogName; } @@ -57,16 +63,27 @@ public String getRemoteCatalogName() { @Override public @Nonnull Map asIcebergCatalogProperties( - UserSecretsManager secretsManager) { + UserSecretsManager secretsManager, PolarisCredentialManager credentialManager) { HashMap properties = new HashMap<>(); properties.put(CatalogProperties.URI, getUri()); if (getRemoteCatalogName() != null) { properties.put(CatalogProperties.WAREHOUSE_LOCATION, getRemoteCatalogName()); } - properties.putAll(getAuthenticationParameters().asIcebergCatalogProperties(secretsManager)); + properties.putAll( + getAuthenticationParameters() + .asIcebergCatalogProperties(secretsManager, credentialManager)); + credentialManager + .getConnectionCredentials(getServiceIdentity(), getAuthenticationParameters()) + .forEach((key, value) -> properties.put(key.getPropertyName(), value)); return properties; } + @Override + public ConnectionConfigInfoDpo withServiceIdentity(ServiceIdentityInfoDpo serviceIdentityInfo) { + return new IcebergRestConnectionConfigInfoDpo( + getUri(), getAuthenticationParameters(), serviceIdentityInfo, getRemoteCatalogName()); + } + @Override public ConnectionConfigInfo asConnectionConfigInfoModel() { return IcebergRestConnectionConfigInfo.builder() @@ -75,6 +92,10 @@ public ConnectionConfigInfo asConnectionConfigInfoModel() { .setRemoteCatalogName(getRemoteCatalogName()) .setAuthenticationParameters( getAuthenticationParameters().asAuthenticationParametersModel()) + .setServiceIdentity( + Optional.ofNullable(getServiceIdentity()) + .map(ServiceIdentityInfoDpo::asServiceIdentityInfoModel) + .orElse(null)) .build(); } @@ -85,6 +106,7 @@ public String toString() { .add("uri", getUri()) .add("remoteCatalogName", getRemoteCatalogName()) .add("authenticationParameters", getAuthenticationParameters().toString()) + .add("serviceIdentity", getServiceIdentity()) .toString(); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityType.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityType.java new file mode 100644 index 0000000000..c05e4573e8 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityType.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.identity; + +import jakarta.annotation.Nonnull; +import java.util.Arrays; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; + +/** + * The internal persistence-object counterpart to ServiceIdentityTypeInfo.ServiceIdentityTypeEnum + * defined in the API model. We define integer type codes in this enum for better compatibility + * within persisted data in case the names of enum types are ever changed in place. + * + *

Important: Codes must be kept in-sync with JsonSubTypes annotated within {@link + * ServiceIdentityInfoDpo}. + */ +public enum ServiceIdentityType { + NULL_TYPE(0), + AWS_IAM(1), + ; + + private static final ServiceIdentityType[] REVERSE_MAPPING_ARRAY; + + static { + // find max array size + int maxCode = + Arrays.stream(ServiceIdentityType.values()) + .mapToInt(ServiceIdentityType::getCode) + .max() + .orElse(0); + + // allocate mapping array + REVERSE_MAPPING_ARRAY = new ServiceIdentityType[maxCode + 1]; + + // populate mapping array + for (ServiceIdentityType authType : ServiceIdentityType.values()) { + REVERSE_MAPPING_ARRAY[authType.code] = authType; + } + } + + private final int code; + + ServiceIdentityType(int code) { + this.code = code; + } + + /** + * Given the code associated with the type, return the associated ServiceIdentityType. Return + * NULL_TYPE if not found + * + * @param identityTypeCode code associated to the service identity type + * @return ServiceIdentityType corresponding to that code or null if mapping not found + */ + public static @Nonnull ServiceIdentityType fromCode(int identityTypeCode) { + // ensure it is within bounds + if (identityTypeCode < 0 || identityTypeCode >= REVERSE_MAPPING_ARRAY.length) { + return NULL_TYPE; + } + + // get value + return REVERSE_MAPPING_ARRAY[identityTypeCode]; + } + + public int getCode() { + return this.code; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/AwsIamServiceIdentityInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/AwsIamServiceIdentityInfoDpo.java new file mode 100644 index 0000000000..0f0f56d7ae --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/AwsIamServiceIdentityInfoDpo.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.identity.dpo; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.regex.Pattern; +import org.apache.polaris.core.admin.model.AwsIamServiceIdentityInfo; +import org.apache.polaris.core.admin.model.ServiceIdentityInfo; +import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.secrets.ServiceSecretReference; + +public class AwsIamServiceIdentityInfoDpo extends ServiceIdentityInfoDpo { + + // Technically, it should be ^arn:(aws|aws-cn|aws-us-gov):iam::(\d{12}):(user|role)/.+$, + @JsonIgnore + public static final String ARN_PATTERN = "^arn:(aws|aws-us-gov):iam::(\\d{12}):(user|role)/.+$"; + + /** AWS IAM role or user ARN that represents the polaris service identity */ + @JsonProperty(value = "iamArn") + private final String iamArn; + + @JsonCreator + public AwsIamServiceIdentityInfoDpo( + @JsonProperty(value = "identityInfoReference", required = false) @Nullable + ServiceSecretReference identityInfoReference, + @JsonProperty(value = "iamArn", required = true) @Nonnull String iamArn) { + this(identityInfoReference, iamArn, true); + } + + protected AwsIamServiceIdentityInfoDpo( + @Nullable ServiceSecretReference identityInfoReference, + @Nonnull String iamArn, + boolean validateArn) { + super(ServiceIdentityType.AWS_IAM.getCode(), identityInfoReference); + this.iamArn = iamArn; + if (validateArn) { + validateArn(iamArn); + } + } + + public String getIamArn() { + return iamArn; + } + + @Override + public @Nonnull ServiceIdentityInfo asServiceIdentityInfoModel() { + return AwsIamServiceIdentityInfo.builder() + .setIdentityType(ServiceIdentityInfo.IdentityTypeEnum.AWS_IAM) + .setIamArn(getIamArn()) + .build(); + } + + /** + * Validates the ARN format. Throws an exception if the ARN is invalid. + * + * @param arn The ARN to validate. + * @throws IllegalArgumentException if the ARN is invalid. + */ + public static void validateArn(String arn) { + if (arn == null || arn.isEmpty()) { + throw new IllegalArgumentException("ARN cannot be null or empty"); + } + // specifically throw errors for China + if (arn.contains("aws-cn")) { + throw new IllegalArgumentException("AWS China is temporarily not supported"); + } + if (!Pattern.matches(ARN_PATTERN, arn)) { + throw new IllegalArgumentException("Invalid role ARN format"); + } + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("identityTypeCode", getIdentityTypeCode()) + .add("iamArn", getIamArn()) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/ServiceIdentityInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/ServiceIdentityInfoDpo.java new file mode 100644 index 0000000000..9eacf10a4e --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/ServiceIdentityInfoDpo.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.identity.dpo; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import org.apache.polaris.core.admin.model.ServiceIdentityInfo; +import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.secrets.ServiceSecretReference; + +/** + * The internal persistence-object counterpart to ServiceIdentityInfo defined in the API model. + * Important: JsonSubTypes must be kept in sync with {@link ServiceIdentityType}. + */ +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.EXISTING_PROPERTY, + property = "identityTypeCode") +@JsonSubTypes({@JsonSubTypes.Type(value = AwsIamServiceIdentityInfoDpo.class, name = "1")}) +public abstract class ServiceIdentityInfoDpo { + + @JsonProperty(value = "identityTypeCode") + private final int identityTypeCode; + + @JsonProperty(value = "identityInfoReference") + private final ServiceSecretReference identityInfoReference; + + public ServiceIdentityInfoDpo( + @JsonProperty(value = "identityTypeCode", required = true) int identityTypeCode, + @JsonProperty(value = "identityInfoReference", required = false) @Nullable + ServiceSecretReference identityInfoReference) { + this.identityTypeCode = identityTypeCode; + this.identityInfoReference = identityInfoReference; + } + + public int getIdentityTypeCode() { + return identityTypeCode; + } + + @JsonIgnore + public ServiceIdentityType getIdentityType() { + return ServiceIdentityType.fromCode(identityTypeCode); + } + + @JsonProperty + public ServiceSecretReference getIdentityInfoReference() { + return identityInfoReference; + } + + public abstract @Nonnull ServiceIdentityInfo asServiceIdentityInfoModel(); + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("identityTypeCode", getIdentityTypeCode()) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/SecretReference.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/SecretReference.java new file mode 100644 index 0000000000..ed95c53e12 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/SecretReference.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.secrets; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Represents a "wrapped reference" to a secret that holds an identifier to retrieve possibly + * remotely-stored secret material, along with an open-ended "referencePayload" that is specific to + * an implementation of the secret storage and which is needed "unwrap" the actual secret in + * combination with whatever is stored in the remote secrets storage. + * + *

Example scenarios: + * + *

If an implementation simply stores secrets directly in the secrets manager, the + * referencePayload may be empty and "unwrapping" would be a simple identity/no-op transformation. + * + *

If tampering or corruption of secrets in the secrets manager presents a unique threat, an + * implementation may use the referencePayload to ensure data integrity of the secret by storing a + * checksum or hash of the stored secret. + * + *

If the system must protect against independent exfiltration/attacks on a dedicated secrets + * manager and the core persistence database, the referencePayload may be used to coordinate + * secondary encryption keys such that the original secret can only be fully "unwrapped" given both + * the stored "secret material" as well as the referencePayload and any associated keys used for + * encryption. + */ +public class SecretReference { + @JsonProperty(value = "urn") + private final String urn; + + @JsonProperty(value = "referencePayload") + private final Map referencePayload; + + /** + * @param urn A string which should be self-sufficient to retrieve whatever secret material that + * is stored in the remote secret store. + * @param referencePayload Optionally, any additional information that is necessary to fully + * reconstitute the original secret based on what is retrieved by the {@code urn}; this + * payload may include hashes/checksums, encryption key ids, OTP encryption keys, additional + * protocol/version specifiers, etc., which are implementation-specific. + */ + public SecretReference( + @JsonProperty(value = "urn", required = true) @Nonnull String urn, + @JsonProperty(value = "referencePayload") @Nullable Map referencePayload) { + this.urn = urn; + this.referencePayload = Objects.requireNonNullElse(referencePayload, new HashMap<>()); + } + + public @Nonnull String getUrn() { + return urn; + } + + public @Nonnull Map getReferencePayload() { + return referencePayload; + } + + @Override + public int hashCode() { + return Objects.hash(getUrn(), getReferencePayload()); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof SecretReference)) { + return false; + } + SecretReference that = (SecretReference) obj; + return Objects.equals(this.getUrn(), that.getUrn()) + && Objects.equals(this.getReferencePayload(), that.getReferencePayload()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("urn", getUrn()) + .add("referencePayload", String.format("", getReferencePayload().size())) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/ServiceSecretReference.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/ServiceSecretReference.java new file mode 100644 index 0000000000..2ab37feea0 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/ServiceSecretReference.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.secrets; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Map; + +/** + * Represents a "wrapped reference" to a service-owned secret that holds an identifier to retrieve + * possibly remotely-stored secret material, along with an open-ended "referencePayload" that is + * specific to an implementation of the secret storage and which is needed "unwrap" the actual + * secret in combination with whatever is stored in the remote secrets storage. + */ +public class ServiceSecretReference extends SecretReference { + /** + * @param urn A string which should be self-sufficient to retrieve whatever secret material that + * is stored in the remote secret store. e.g., + * 'urn:polaris-service-secret:<service-manager-type>:<type-specific-identifier> + * @param referencePayload Optionally, any additional information that is necessary to fully + * reconstitute the original secret based on what is retrieved by the {@code urn}; this + * payload may include hashes/checksums, encryption key ids, OTP encryption keys, additional + * protocol/version specifiers, etc., which are implementation-specific. + */ + public ServiceSecretReference( + @JsonProperty(value = "urn", required = true) @Nonnull String urn, + @JsonProperty(value = "referencePayload") @Nullable Map referencePayload) { + super(urn, referencePayload); + // TODO: Add better/standardized parsing and validation of URN syntax + Preconditions.checkArgument( + urn.startsWith("urn:polaris-service-secret:") && urn.split(":").length >= 5, + "Invalid secret URN '%s'; must be of the form " + + "'urn:polaris-service-secret::'", + urn); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ServiceSecretReference && super.equals(obj); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java index 7181acb041..71a7839df2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java @@ -20,42 +20,18 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; -import java.util.HashMap; import java.util.Map; -import java.util.Objects; /** * Represents a "wrapped reference" to a user-owned secret that holds an identifier to retrieve * possibly remotely-stored secret material, along with an open-ended "referencePayload" that is * specific to an implementation of the secret storage and which is needed "unwrap" the actual * secret in combination with whatever is stored in the remote secrets storage. - * - *

Example scenarios: - * - *

If an implementation simply stores secrets directly in the secrets manager, the - * referencePayload may be empty and "unwrapping" would be a simple identity/no-op transformation. - * - *

If tampering or corruption of secrets in the secrets manager presents a unique threat, an - * implementation may use the referencePayload to ensure data integrity of the secret by storing a - * checksum or hash of the stored secret. - * - *

If the system must protect against independent exfiltration/attacks on a dedicated secrets - * manager and the core persistence database, the referencePayload may be used to coordinate - * secondary encryption keys such that the original secret can only be fully "unwrapped" given both - * the stored "secret material" as well as the referencePayload and any associated keys used for - * encryption. */ -public class UserSecretReference { - @JsonProperty(value = "urn") - private final String urn; - - @JsonProperty(value = "referencePayload") - private final Map referencePayload; - +public class UserSecretReference extends SecretReference { /** * @param urn A string which should be self-sufficient to retrieve whatever secret material that * is stored in the remote secret store and also to identify an implementation of the @@ -70,14 +46,13 @@ public class UserSecretReference { public UserSecretReference( @JsonProperty(value = "urn", required = true) @Nonnull String urn, @JsonProperty(value = "referencePayload") @Nullable Map referencePayload) { + super(urn, referencePayload); // TODO: Add better/standardized parsing and validation of URN syntax Preconditions.checkArgument( urn.startsWith("urn:polaris-secret:") && urn.split(":").length >= 4, "Invalid secret URN '%s'; must be of the form " + "'urn:polaris-secret::'", urn); - this.urn = urn; - this.referencePayload = Objects.requireNonNullElse(referencePayload, new HashMap<>()); } /** @@ -89,37 +64,11 @@ public UserSecretReference( @JsonIgnore public String getUserSecretManagerTypeFromUrn() { // TODO: Add better/standardized parsing and validation of URN syntax - return urn.split(":")[2]; - } - - public @Nonnull String getUrn() { - return urn; - } - - public @Nonnull Map getReferencePayload() { - return referencePayload; - } - - @Override - public int hashCode() { - return Objects.hash(getUrn(), getReferencePayload()); + return getUrn().split(":")[2]; } @Override public boolean equals(Object obj) { - if (obj == null || !(obj instanceof UserSecretReference)) { - return false; - } - UserSecretReference that = (UserSecretReference) obj; - return Objects.equals(this.getUrn(), that.getUrn()) - && Objects.equals(this.getReferencePayload(), that.getReferencePayload()); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("urn", getUrn()) - .add("referencePayload", String.format("", getReferencePayload().size())) - .toString(); + return obj instanceof UserSecretReference && super.equals(obj); } } From a35b7ddc17bce5d55f49e992cbb29b48db8f36e3 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Tue, 27 May 2025 17:05:28 -0700 Subject: [PATCH 02/15] Add server configuration classes for service identity info --- ...rkusRealmServiceIdentityConfiguration.java | 34 +++++ .../QuarkusServiceIdentityConfiguration.java | 37 +++++ ...sServiceIdentityRegistryConfiguration.java | 35 +++++ .../AwsIamServiceIdentityConfiguration.java | 47 +++++++ ...DefaultServiceIdentityRegistryFactory.java | 128 ++++++++++++++++++ .../RealmServiceIdentityConfiguration.java | 32 +++++ ...esolvableServiceIdentityConfiguration.java | 27 ++++ .../ServiceIdentityConfiguration.java | 39 ++++++ 8 files changed, 379 insertions(+) create mode 100644 quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusRealmServiceIdentityConfiguration.java create mode 100644 quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityConfiguration.java create mode 100644 quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityRegistryConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/AwsIamServiceIdentityConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/RealmServiceIdentityConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/ResolvableServiceIdentityConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/ServiceIdentityConfiguration.java diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusRealmServiceIdentityConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusRealmServiceIdentityConfiguration.java new file mode 100644 index 0000000000..f5cf2f94f8 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusRealmServiceIdentityConfiguration.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.identity; + +import io.smallrye.config.WithName; +import java.util.Optional; +import org.apache.polaris.service.identity.AwsIamServiceIdentityConfiguration; +import org.apache.polaris.service.identity.RealmServiceIdentityConfiguration; + +public interface QuarkusRealmServiceIdentityConfiguration + extends RealmServiceIdentityConfiguration { + @Override + @WithName("aws-iam") + Optional awsIamServiceIdentity(); + + interface QuarkusAwsIamServiceIdentityConfiguration extends AwsIamServiceIdentityConfiguration {} +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityConfiguration.java new file mode 100644 index 0000000000..8a6fb9385b --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityConfiguration.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.identity; + +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefaults; +import io.smallrye.config.WithParentName; +import io.smallrye.config.WithUnnamedKey; +import java.util.Map; +import org.apache.polaris.service.identity.ServiceIdentityConfiguration; + +@ConfigMapping(prefix = "polaris.service-identity") +public interface QuarkusServiceIdentityConfiguration + extends ServiceIdentityConfiguration { + @WithParentName + @WithUnnamedKey(DEFAULT_REALM_KEY) + @WithDefaults + @Override + Map realms(); +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityRegistryConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityRegistryConfiguration.java new file mode 100644 index 0000000000..38038a3ec5 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityRegistryConfiguration.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.identity; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import org.apache.polaris.core.identity.ServiceIdentityRegistryFactory; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.service-identity.registry") +public interface QuarkusServiceIdentityRegistryConfiguration { + + /** + * The type of the ServiceIdentityRegistryFactory to use. This is the {@link + * ServiceIdentityRegistryFactory} identifier. + */ + String type(); +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/AwsIamServiceIdentityConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/AwsIamServiceIdentityConfiguration.java new file mode 100644 index 0000000000..cba3bf9b20 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/AwsIamServiceIdentityConfiguration.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.identity; + +import java.util.Optional; +import org.apache.polaris.core.identity.resolved.ResolvedAwsIamServiceIdentity; + +public interface AwsIamServiceIdentityConfiguration extends ResolvableServiceIdentityConfiguration { + String iamArn(); + + Optional accessKeyId(); + + Optional secretAccessKey(); + + Optional sessionToken(); + + @Override + default Optional resolve() { + if (iamArn() == null) { + return Optional.empty(); + } else { + return Optional.of( + new ResolvedAwsIamServiceIdentity( + iamArn(), + accessKeyId().orElse(null), + secretAccessKey().orElse(null), + sessionToken().orElse(null))); + } + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java b/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java new file mode 100644 index 0000000000..821a47fefc --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.identity; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.util.EnumMap; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.identity.DefaultServiceIdentityRegistry; +import org.apache.polaris.core.identity.ServiceIdentityRegistry; +import org.apache.polaris.core.identity.ServiceIdentityRegistryFactory; +import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; +import org.apache.polaris.core.secrets.ServiceSecretReference; + +@ApplicationScoped +@Identifier("default") +public class DefaultServiceIdentityRegistryFactory implements ServiceIdentityRegistryFactory { + private static final String DEFAULT_REALM_KEY = ServiceIdentityConfiguration.DEFAULT_REALM_KEY; + private static final String IDENTITY_INFO_REFERENCE_URN_FORMAT = + "urn:polaris-service-secret:default-identity-registry:%s:%s"; + + private final Map realmServiceIdentityRegistries; + + @Inject + public DefaultServiceIdentityRegistryFactory( + ServiceIdentityConfiguration serviceIdentityConfiguration) { + realmServiceIdentityRegistries = + serviceIdentityConfiguration.realms().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, // realm identifier + entry -> { + RealmServiceIdentityConfiguration realmConfig = entry.getValue(); + + // Resolve all the service identities for the realm + EnumMap resolvedIdentities = + realmConfig.serviceIdentityConfigurations().stream() + .map(ResolvableServiceIdentityConfiguration::resolve) + .flatMap(Optional::stream) + .peek( + // Set the identity info reference for each resolved identity + identity -> + identity.setIdentityInfoReference( + buildIdentityInfoReference( + entry.getKey(), identity.getIdentityType()))) + .collect( + // Collect to an EnumMap, grouping by ServiceIdentityType + Collectors.toMap( + ResolvedServiceIdentity::getIdentityType, + identity -> identity, + (a, b) -> b, + () -> new EnumMap<>(ServiceIdentityType.class))); + return new DefaultServiceIdentityRegistry(resolvedIdentities); + })); + + if (!realmServiceIdentityRegistries.containsKey(DEFAULT_REALM_KEY)) { + // If no default realm is defined, create an empty registry + realmServiceIdentityRegistries.put( + DEFAULT_REALM_KEY, + new DefaultServiceIdentityRegistry(new EnumMap<>(ServiceIdentityType.class))); + } + } + + public DefaultServiceIdentityRegistryFactory() { + this(new DefaultServiceIdentityRegistry(new EnumMap<>(ServiceIdentityType.class))); + } + + public DefaultServiceIdentityRegistryFactory( + DefaultServiceIdentityRegistry defaultServiceIdentityRegistry) { + this(Map.of(DEFAULT_REALM_KEY, defaultServiceIdentityRegistry)); + } + + public DefaultServiceIdentityRegistryFactory( + Map realmServiceIdentityRegistries) { + this.realmServiceIdentityRegistries = realmServiceIdentityRegistries; + + if (!realmServiceIdentityRegistries.containsKey(DEFAULT_REALM_KEY)) { + // If no default realm is defined, create an empty registry + realmServiceIdentityRegistries.put( + DEFAULT_REALM_KEY, + new DefaultServiceIdentityRegistry(new EnumMap<>(ServiceIdentityType.class))); + } + } + + @Override + public ServiceIdentityRegistry getOrCreateServiceIdentityRegistry(RealmContext realmContext) { + return getServiceIdentityRegistryForRealm(realmContext); + } + + protected DefaultServiceIdentityRegistry getServiceIdentityRegistryForRealm( + RealmContext realmContext) { + return getServiceIdentityRegistryForRealm(realmContext.getRealmIdentifier()); + } + + protected DefaultServiceIdentityRegistry getServiceIdentityRegistryForRealm( + String realmIdentifier) { + return realmServiceIdentityRegistries.getOrDefault( + realmIdentifier, realmServiceIdentityRegistries.get(DEFAULT_REALM_KEY)); + } + + private ServiceSecretReference buildIdentityInfoReference( + String realm, ServiceIdentityType type) { + // urn:polaris-service-secret:default-identity-registry:: + return new ServiceSecretReference( + IDENTITY_INFO_REFERENCE_URN_FORMAT.formatted(realm, type.name()), Map.of()); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/RealmServiceIdentityConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/RealmServiceIdentityConfiguration.java new file mode 100644 index 0000000000..0c61d7ff6a --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/RealmServiceIdentityConfiguration.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.identity; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +public interface RealmServiceIdentityConfiguration { + Optional awsIamServiceIdentity(); + + default List serviceIdentityConfigurations() { + return Stream.of(awsIamServiceIdentity()).flatMap(Optional::stream).toList(); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/ResolvableServiceIdentityConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/ResolvableServiceIdentityConfiguration.java new file mode 100644 index 0000000000..14b7651075 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/ResolvableServiceIdentityConfiguration.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.identity; + +import java.util.Optional; +import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; + +public interface ResolvableServiceIdentityConfiguration { + Optional resolve(); +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/ServiceIdentityConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/ServiceIdentityConfiguration.java new file mode 100644 index 0000000000..ec5e491476 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/ServiceIdentityConfiguration.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.identity; + +import java.util.Map; +import org.apache.polaris.core.context.RealmContext; + +public interface ServiceIdentityConfiguration { + String DEFAULT_REALM_KEY = ""; + + Map realms(); + + default R forRealm(RealmContext realmContext) { + return forRealm(realmContext.getRealmIdentifier()); + } + + default R forRealm(String realmIdentifier) { + return realms().containsKey(realmIdentifier) + ? realms().get(realmIdentifier) + : realms().get(DEFAULT_REALM_KEY); + } +} From c8fe1b05e0817a70bd4736dcd2ba5b0be1ffc539 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Tue, 27 May 2025 17:07:18 -0700 Subject: [PATCH 03/15] Add Service Identity Registry --- .../DefaultServiceIdentityRegistry.java | 63 +++++++++++++ .../identity/ServiceIdentityRegistry.java | 35 ++++++++ .../ServiceIdentityRegistryFactory.java | 26 ++++++ .../ResolvedAwsIamServiceIdentity.java | 89 +++++++++++++++++++ .../resolved/ResolvedServiceIdentity.java | 61 +++++++++++++ 5 files changed, 274 insertions(+) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/DefaultServiceIdentityRegistry.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistry.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistryFactory.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedServiceIdentity.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/DefaultServiceIdentityRegistry.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/DefaultServiceIdentityRegistry.java new file mode 100644 index 0000000000..23ab19ddaf --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/DefaultServiceIdentityRegistry.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.identity; + +import java.util.EnumMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; + +public class DefaultServiceIdentityRegistry implements ServiceIdentityRegistry { + + private final EnumMap resolvedServiceIdentities; + private final Map referenceToResolvedServiceIdentity; + + public DefaultServiceIdentityRegistry( + EnumMap serviceIdentities) { + this.resolvedServiceIdentities = serviceIdentities; + this.referenceToResolvedServiceIdentity = + serviceIdentities.values().stream() + .collect( + Collectors.toMap( + identity -> identity.getIdentityInfoReference().getUrn(), + identity -> identity)); + } + + @Override + public ServiceIdentityInfoDpo assignServiceIdentity(ServiceIdentityType serviceIdentityType) { + ResolvedServiceIdentity resolvedServiceIdentity = + resolvedServiceIdentities.get(serviceIdentityType); + if (resolvedServiceIdentity == null) { + throw new IllegalArgumentException( + "Service identity type not supported: " + serviceIdentityType); + } + return resolvedServiceIdentity.asServiceIdentityInfoDpo(); + } + + @Override + public ResolvedServiceIdentity resolveServiceIdentity( + ServiceIdentityInfoDpo serviceIdentityInfo) { + ResolvedServiceIdentity resolvedServiceIdentity = + referenceToResolvedServiceIdentity.get( + serviceIdentityInfo.getIdentityInfoReference().getUrn()); + return resolvedServiceIdentity; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistry.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistry.java new file mode 100644 index 0000000000..4177521d3c --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistry.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.identity; + +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; + +public interface ServiceIdentityRegistry { + ServiceIdentityInfoDpo assignServiceIdentity(ServiceIdentityType serviceIdentityType); + + /** + * Resolves the service identity based on the provided service identity information. + * + * @param serviceIdentityInfo The service identity information to resolve. + * @return The resolved service identity. + */ + ResolvedServiceIdentity resolveServiceIdentity(ServiceIdentityInfoDpo serviceIdentityInfo); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistryFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistryFactory.java new file mode 100644 index 0000000000..7b20e42c5e --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistryFactory.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.identity; + +import org.apache.polaris.core.context.RealmContext; + +public interface ServiceIdentityRegistryFactory { + ServiceIdentityRegistry getOrCreateServiceIdentityRegistry(RealmContext realmContext); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java new file mode 100644 index 0000000000..e3e52a974a --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.identity.resolved; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import jakarta.annotation.Nonnull; +import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.identity.dpo.AwsIamServiceIdentityInfoDpo; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; + +public class ResolvedAwsIamServiceIdentity extends ResolvedServiceIdentity { + private final String iamArn; + private final String accessKeyId; + private final String secretAccessKey; + private final String sessionToken; + + public ResolvedAwsIamServiceIdentity( + String iamArn, String accessKeyId, String secretAccessKey, String sessionToken) { + super(ServiceIdentityType.AWS_IAM); + this.iamArn = iamArn; + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; + this.sessionToken = sessionToken; + } + + public String getIamArn() { + return iamArn; + } + + public String getAccessKeyId() { + return accessKeyId; + } + + public String getSecretAccessKey() { + return secretAccessKey; + } + + public String getSessionToken() { + return sessionToken; + } + + @Nonnull + @Override + public ServiceIdentityInfoDpo asServiceIdentityInfoDpo() { + return new AwsIamServiceIdentityInfoDpo(getIdentityInfoReference(), getIamArn()); + } + + public Supplier stsClientSupplier() { + return Suppliers.memoize( + () -> { + StsClientBuilder stsClientBuilder = StsClient.builder(); + if (getAccessKeyId() != null && getSecretAccessKey() != null) { + StaticCredentialsProvider awsCredentialsProvider = + StaticCredentialsProvider.create( + AwsBasicCredentials.create(getAccessKeyId(), getSecretAccessKey())); + if (getSessionToken() != null) { + awsCredentialsProvider = + StaticCredentialsProvider.create( + AwsSessionCredentials.create( + getAccessKeyId(), getSecretAccessKey(), getSessionToken())); + } + stsClientBuilder.credentialsProvider(awsCredentialsProvider); + } + return stsClientBuilder.build(); + }); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedServiceIdentity.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedServiceIdentity.java new file mode 100644 index 0000000000..ad981570c1 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedServiceIdentity.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.identity.resolved; + +import jakarta.annotation.Nonnull; +import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +import org.apache.polaris.core.secrets.ServiceSecretReference; +import software.amazon.awssdk.annotations.NotNull; + +/** + * Represents a resolved service identity. + * + *

This class is used to represent the identity of a service after it has been resolved. It + * contains the type of the identity and any additional information for the service identity. E.g., + * The credential of the service identity. + */ +public abstract class ResolvedServiceIdentity { + private final ServiceIdentityType identityType; + private ServiceSecretReference identityInfoReference; + + public ResolvedServiceIdentity(ServiceIdentityType identityType) { + this(identityType, null); + } + + public ResolvedServiceIdentity( + ServiceIdentityType identityType, ServiceSecretReference identityInfoReference) { + this.identityType = identityType; + this.identityInfoReference = identityInfoReference; + } + + public @NotNull ServiceIdentityType getIdentityType() { + return identityType; + } + + public @Nonnull ServiceSecretReference getIdentityInfoReference() { + return identityInfoReference; + } + + public void setIdentityInfoReference(@NotNull ServiceSecretReference identityInfoReference) { + this.identityInfoReference = identityInfoReference; + } + + public abstract @Nonnull ServiceIdentityInfoDpo asServiceIdentityInfoDpo(); +} From 6eb43e157aff4fb1a8d2f5408cb8b17867e7bb40 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Tue, 27 May 2025 17:12:08 -0700 Subject: [PATCH 04/15] Add Entity mutators to inject Service Identity --- .../polaris/core/PolarisCallContext.java | 20 +++++- .../mutation/EntityMutationEngine.java | 26 +++++++ .../LocalPolarisMetaStoreManagerFactory.java | 15 ++-- .../TransactionalMetaStoreManagerImpl.java | 4 ++ .../mutation/NoOpEntityMutationEngine.java | 30 ++++++++ .../src/main/resources/application.properties | 10 ++- quarkus/service/build.gradle.kts | 1 + .../config/ProductionReadinessChecks.java | 19 +++++ .../quarkus/config/QuarkusProducers.java | 58 ++++++++++++++- .../CatalogEntityConnectionConfigMutator.java | 70 +++++++++++++++++++ .../identity/mutation/NoOpEntityMutator.java | 39 +++++++++++ service/common/build.gradle.kts | 1 + .../service/admin/PolarisAdminService.java | 5 ++ .../mutation/EntityMutationEngineImpl.java | 55 +++++++++++++++ .../identity/mutation/EntityMutator.java | 28 ++++++++ 15 files changed, 373 insertions(+), 8 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java create mode 100644 polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java create mode 100644 quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java create mode 100644 quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java index 9e5a7a8b42..f39146c9e5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java @@ -24,6 +24,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.BasePersistence; /** @@ -40,6 +41,8 @@ public class PolarisCallContext implements CallContext { private final PolarisConfigurationStore configurationStore; + private final EntityMutationEngine entityMutationEngine; + private final Clock clock; // will make it final once we remove deprecated constructor @@ -50,22 +53,26 @@ public PolarisCallContext( @Nonnull BasePersistence metaStore, @Nonnull PolarisDiagnostics diagServices, @Nonnull PolarisConfigurationStore configurationStore, + @Nonnull EntityMutationEngine entityMutationEngine, @Nonnull Clock clock) { this.realmContext = realmContext; this.metaStore = metaStore; this.diagServices = diagServices; this.configurationStore = configurationStore; + this.entityMutationEngine = entityMutationEngine; this.clock = clock; } public PolarisCallContext( @Nonnull RealmContext realmContext, @Nonnull BasePersistence metaStore, - @Nonnull PolarisDiagnostics diagServices) { + @Nonnull PolarisDiagnostics diagServices, + @Nonnull EntityMutationEngine entityMutationEngine) { this.realmContext = realmContext; this.metaStore = metaStore; this.diagServices = diagServices; this.configurationStore = new PolarisConfigurationStore() {}; + this.entityMutationEngine = entityMutationEngine; this.clock = Clock.system(ZoneId.systemDefault()); } @@ -81,6 +88,10 @@ public PolarisConfigurationStore getConfigurationStore() { return configurationStore; } + public EntityMutationEngine getEntityMutationEngine() { + return entityMutationEngine; + } + public Clock getClock() { return clock; } @@ -105,6 +116,11 @@ public PolarisCallContext copy() { String realmId = this.realmContext.getRealmIdentifier(); RealmContext realmContext = () -> realmId; return new PolarisCallContext( - realmContext, this.metaStore, this.diagServices, this.configurationStore, this.clock); + realmContext, + this.metaStore, + this.diagServices, + this.configurationStore, + this.entityMutationEngine, + this.clock); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java new file mode 100644 index 0000000000..9094a09fc3 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.identity.mutation; + +import org.apache.polaris.core.entity.PolarisBaseEntity; + +public interface EntityMutationEngine { + PolarisBaseEntity applyMutations(PolarisBaseEntity entity); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 1cfa89d0f0..c9730bcdc2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -32,6 +32,7 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.InMemoryEntityCache; @@ -63,10 +64,13 @@ public abstract class LocalPolarisMetaStoreManagerFactory LoggerFactory.getLogger(LocalPolarisMetaStoreManagerFactory.class); private final PolarisDiagnostics diagnostics; + private final EntityMutationEngine entityMutationEngine; private boolean bootstrap; - protected LocalPolarisMetaStoreManagerFactory(@Nonnull PolarisDiagnostics diagnostics) { + protected LocalPolarisMetaStoreManagerFactory( + @Nonnull PolarisDiagnostics diagnostics, @Nonnull EntityMutationEngine entityMutationEngine) { this.diagnostics = diagnostics; + this.entityMutationEngine = entityMutationEngine; } protected abstract StoreType createBackingStore(@Nonnull PolarisDiagnostics diagnostics); @@ -134,7 +138,8 @@ public Map purgeRealms(Iterable realms) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); TransactionalPersistence session = getOrCreateSessionSupplier(realmContext).get(); - PolarisCallContext callContext = new PolarisCallContext(realmContext, session, diagServices); + PolarisCallContext callContext = + new PolarisCallContext(realmContext, session, diagServices, entityMutationEngine); BaseResult result = metaStoreManager.purge(callContext); results.put(realm, result); @@ -207,7 +212,8 @@ private PrincipalSecretsResult bootstrapServiceAndCreatePolarisPrincipalForRealm new PolarisCallContext( realmContext, sessionSupplierMap.get(realmContext.getRealmIdentifier()).get(), - diagServices); + diagServices, + entityMutationEngine); if (CallContext.getCurrentContext() == null) { CallContext.setCurrentContext(polarisContext); } @@ -256,7 +262,8 @@ private void checkPolarisServiceBootstrappedForRealm( new PolarisCallContext( realmContext, sessionSupplierMap.get(realmContext.getRealmIdentifier()).get(), - diagServices); + diagServices, + entityMutationEngine); if (CallContext.getCurrentContext() == null) { CallContext.setCurrentContext(polarisContext); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index 0d6111d54b..6e19e74879 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -473,6 +473,10 @@ private void revokeGrantRecord( ms.persistStorageIntegrationIfNeededInCurrentTxn(callCtx, catalog, integration); + if (callCtx.getEntityMutationEngine() != null) { + catalog = callCtx.getEntityMutationEngine().applyMutations(catalog); + } + // now create and persist new catalog entity this.persistNewEntity(callCtx, ms, catalog); diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java new file mode 100644 index 0000000000..58f0f4525f --- /dev/null +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.identity.mutation; + +import org.apache.polaris.core.entity.PolarisBaseEntity; + +public class NoOpEntityMutationEngine implements EntityMutationEngine { + + @Override + public PolarisBaseEntity applyMutations(PolarisBaseEntity entity) { + return entity; + } +} diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index 592501f76d..f83338f4e1 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -110,7 +110,7 @@ polaris.realm-context.require-header=false polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"] -# polaris.features."ENABLE_CATALOG_FEDERATION"=true +polaris.features."ENABLE_CATALOG_FEDERATION"=true polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] # realm overrides @@ -188,6 +188,14 @@ polaris.oidc.principal-roles-mapper.type=default # polaris.storage.gcp.token=token # polaris.storage.gcp.lifespan=PT1H +# Polaris Service Identity Config +polaris.service-identity.registry.type=default +polaris.service-identity.aws-iam.iam-arn=arn:aws:iam::123456789012:user/polaris-iam-user +polaris.service-identity.my-realm.aws-iam.iam-arn=arn:aws:iam::123456789012:user/polaris-iam-user + +# Polaris Credential Manager Config +polaris.credential-manager.type=default + quarkus.arc.ignored-split-packages=\ org.apache.polaris.service.catalog.api,\ org.apache.polaris.service.catalog.api.impl,\ diff --git a/quarkus/service/build.gradle.kts b/quarkus/service/build.gradle.kts index 01af01d3c4..28ef491abd 100644 --- a/quarkus/service/build.gradle.kts +++ b/quarkus/service/build.gradle.kts @@ -99,6 +99,7 @@ dependencies { } testImplementation(project(":polaris-api-management-model")) + testImplementation(testFixtures(project(":polaris-core"))) testImplementation(testFixtures(project(":polaris-service-common"))) testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java index ef2c1813e9..375952ad38 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java @@ -46,6 +46,7 @@ import org.apache.polaris.service.events.TestPolarisEventListener; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.quarkus.auth.QuarkusAuthenticationConfiguration; +import org.apache.polaris.service.quarkus.identity.QuarkusServiceIdentityConfiguration; import org.eclipse.microprofile.config.Config; import org.eclipse.microprofile.config.ConfigValue; import org.slf4j.Logger; @@ -207,6 +208,24 @@ public ProductionReadinessCheck checkPolarisEventListener( return ProductionReadinessCheck.OK; } + @Produces + public ProductionReadinessCheck checkServiceIdentities( + QuarkusServiceIdentityConfiguration configuration) { + List errors = new ArrayList<>(); + configuration + .realms() + .forEach( + (realm, config) -> { + if (config.awsIamServiceIdentity().isEmpty()) { + errors.add( + Error.of( + "AWS IAM Service identity is not configured.", + "polaris.service-identity.%saws-iam".formatted(authRealmSegment(realm)))); + } + }); + return ProductionReadinessCheck.of(errors); + } + private static String authRealmSegment(String realm) { return realm.equals(QuarkusAuthenticationConfiguration.DEFAULT_REALM_KEY) ? "" : realm + "."; } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 593853c409..1e4f9d780b 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -42,6 +42,11 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.credentials.PolarisCredentialManager; +import org.apache.polaris.core.credentials.PolarisCredentialManagerFactory; +import org.apache.polaris.core.identity.ServiceIdentityRegistry; +import org.apache.polaris.core.identity.ServiceIdentityRegistryFactory; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; @@ -62,6 +67,7 @@ import org.apache.polaris.service.context.RealmContextConfiguration; import org.apache.polaris.service.context.RealmContextResolver; import org.apache.polaris.service.events.PolarisEventListener; +import org.apache.polaris.service.identity.ServiceIdentityConfiguration; import org.apache.polaris.service.quarkus.auth.QuarkusAuthenticationConfiguration; import org.apache.polaris.service.quarkus.auth.QuarkusAuthenticationRealmConfiguration; import org.apache.polaris.service.quarkus.auth.external.tenant.OidcTenantResolver; @@ -69,6 +75,9 @@ import org.apache.polaris.service.quarkus.context.QuarkusRealmContextConfiguration; import org.apache.polaris.service.quarkus.context.RealmContextFilter; import org.apache.polaris.service.quarkus.events.QuarkusPolarisEventListenerConfiguration; +import org.apache.polaris.service.quarkus.identity.QuarkusRealmServiceIdentityConfiguration; +import org.apache.polaris.service.quarkus.identity.QuarkusServiceIdentityConfiguration; +import org.apache.polaris.service.quarkus.identity.QuarkusServiceIdentityRegistryConfiguration; import org.apache.polaris.service.quarkus.persistence.QuarkusPersistenceConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusRateLimiterFilterConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusTokenBucketConfiguration; @@ -126,11 +135,17 @@ public CallContext polarisCallContext( PolarisDiagnostics diagServices, PolarisConfigurationStore configurationStore, MetaStoreManagerFactory metaStoreManagerFactory, + EntityMutationEngine entityMutationEngine, Clock clock) { BasePersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); return new PolarisCallContext( - realmContext, metaStoreSession, diagServices, configurationStore, clock); + realmContext, + metaStoreSession, + diagServices, + configurationStore, + entityMutationEngine, + clock); } // Polaris service beans - selected from @Identifier-annotated beans @@ -169,6 +184,26 @@ public UserSecretsManagerFactory userSecretsManagerFactory( return userSecretsManagerFactories.select(Identifier.Literal.of(config.type())).get(); } + @Produces + public ServiceIdentityConfiguration + serviceIdentityConfiguration(QuarkusServiceIdentityConfiguration config) { + return config; + } + + @Produces + public ServiceIdentityRegistryFactory serviceIdentityRegistryFactory( + QuarkusServiceIdentityRegistryConfiguration config, + @Any Instance serviceIdentityRegistryFactories) { + return serviceIdentityRegistryFactories.select(Identifier.Literal.of(config.type())).get(); + } + + @Produces + public PolarisCredentialManagerFactory credentialManagerFactory( + QuarkusServiceIdentityRegistryConfiguration config, + @Any Instance credentialManagerFactories) { + return credentialManagerFactories.select(Identifier.Literal.of(config.type())).get(); + } + /** * Eagerly initialize the in-memory default realm on startup, so that users can check the * credentials printed to stdout immediately. @@ -338,6 +373,27 @@ public QuarkusAuthenticationRealmConfiguration realmAuthConfig( return config.forRealm(realmContext); } + @Produces + @RequestScoped + public QuarkusRealmServiceIdentityConfiguration realmServiceIdentityConfig( + QuarkusServiceIdentityConfiguration config, RealmContext realmContext) { + return config.forRealm(realmContext); + } + + @Produces + @RequestScoped + public ServiceIdentityRegistry serviceIdentityRegistry( + ServiceIdentityRegistryFactory serviceIdentityRegistryFactory, RealmContext realmContext) { + return serviceIdentityRegistryFactory.getOrCreateServiceIdentityRegistry(realmContext); + } + + @Produces + @RequestScoped + public PolarisCredentialManager polarisCredentialManager( + PolarisCredentialManagerFactory polarisCredentialManagerFactory, RealmContext realmContext) { + return polarisCredentialManagerFactory.getOrCreatePolarisCredentialManager(realmContext); + } + @Produces public ActiveRolesProvider activeRolesProvider( @ConfigProperty(name = "polaris.active-roles-provider.type") String activeRolesProviderType, diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java new file mode 100644 index 0000000000..b859ef221e --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.identity.mutation; + +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; +import org.apache.polaris.core.connection.AuthenticationParametersDpo; +import org.apache.polaris.core.connection.AuthenticationType; +import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.identity.ServiceIdentityRegistry; +import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.service.identity.mutation.EntityMutator; + +@RequestScoped +public class CatalogEntityConnectionConfigMutator implements EntityMutator { + + private static final String MUTATOR_ID = "catalog-connection-config"; + + ServiceIdentityRegistry serviceIdentityRegistry; + + @Inject + CatalogEntityConnectionConfigMutator(ServiceIdentityRegistry serviceIdentityRegistry) { + this.serviceIdentityRegistry = serviceIdentityRegistry; + } + + @Override + public String id() { + return MUTATOR_ID; + } + + @Override + public PolarisBaseEntity apply(PolarisBaseEntity entity) { + if (!(entity instanceof CatalogEntity catalogEntity) || !catalogEntity.isPassthroughFacade()) { + return entity; + } + + ConnectionConfigInfoDpo connectionConfigInfoDpo = catalogEntity.getConnectionConfigInfoDpo(); + AuthenticationParametersDpo authenticationParameters = + connectionConfigInfoDpo.getAuthenticationParameters(); + if (authenticationParameters.getAuthenticationType() == AuthenticationType.SIGV4) { + CatalogEntity.Builder builder = new CatalogEntity.Builder(catalogEntity); + ConnectionConfigInfoDpo injectedConnectionConfigInfoDpo = + connectionConfigInfoDpo.withServiceIdentity( + serviceIdentityRegistry.assignServiceIdentity(ServiceIdentityType.AWS_IAM)); + builder.setConnectionConfigInfoDpo(injectedConnectionConfigInfoDpo); + builder.setEntityVersion(entity.getEntityVersion() + 1); + return builder.build(); + } + return entity; + } +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java new file mode 100644 index 0000000000..13f7783320 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.identity.mutation; + +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.service.identity.mutation.EntityMutator; + +@ApplicationScoped +public class NoOpEntityMutator implements EntityMutator { + private static final String MUTATOR_ID = "no-op"; + + @Override + public String id() { + return MUTATOR_ID; + } + + @Override + public PolarisBaseEntity apply(PolarisBaseEntity entity) { + return entity; + } +} diff --git a/service/common/build.gradle.kts b/service/common/build.gradle.kts index 2f5d958259..b110eb04cd 100644 --- a/service/common/build.gradle.kts +++ b/service/common/build.gradle.kts @@ -103,6 +103,7 @@ dependencies { testFixturesImplementation(project(":polaris-api-management-service")) testFixturesImplementation(project(":polaris-api-iceberg-service")) testFixturesImplementation(project(":polaris-api-catalog-service")) + testFixturesImplementation(testFixtures(project(":polaris-core"))) testFixturesImplementation(libs.jakarta.enterprise.cdi.api) testFixturesImplementation(libs.jakarta.annotation.api) diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 4495d96fd8..ac15b518b7 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -687,6 +687,11 @@ private Map extractSecretReferences( AuthenticationParametersDpo.INLINE_BEARER_TOKEN_REFERENCE_KEY, secretReference); break; } + case SIGV4: + { + // SigV4 authentication is not secret-based + break; + } default: throw new IllegalStateException( "Unsupported authentication type: " diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java new file mode 100644 index 0000000000..86cbd20a13 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.identity.mutation; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Instance; +import jakarta.inject.Inject; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; + +@ApplicationScoped +public class EntityMutationEngineImpl implements EntityMutationEngine { + + private final Instance mutatorInstance; + + @Inject + public EntityMutationEngineImpl(Instance mutatorInstance) { + this.mutatorInstance = mutatorInstance; + } + + @Override + public PolarisBaseEntity applyMutations(PolarisBaseEntity entity) { + PolarisBaseEntity result = entity; + + Map mutatorMap = + mutatorInstance.stream().collect(Collectors.toMap(EntityMutator::id, m -> m)); + + // Apply all mutators to the entity + // TODO: Add a way to control the order of mutators + for (EntityMutator mutator : mutatorMap.values()) { + result = mutator.apply(result); + } + + return result; + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java new file mode 100644 index 0000000000..7254770d86 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.identity.mutation; + +import org.apache.polaris.core.entity.PolarisBaseEntity; + +public interface EntityMutator { + String id(); + + PolarisBaseEntity apply(PolarisBaseEntity entity); +} From f76b0e7dc1bb4181cb57041da76fdaef9b5ce124 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Tue, 27 May 2025 17:16:28 -0700 Subject: [PATCH 05/15] Retrieve Connection Creds via PolarisCredentialManager --- ...pseLinkPolarisMetaStoreManagerFactory.java | 9 +- .../jdbc/JdbcMetaStoreManagerFactory.java | 11 ++- .../DefaultPolarisCredentialManager.java | 93 +++++++++++++++++++ .../credentials/PolarisCredentialManager.java | 32 +++++++ .../PolarisCredentialManagerFactory.java | 32 +++++++ .../ConnectionCredentialProperty.java | 56 +++++++++++ .../PolarisConnectionCredsVendor.java | 29 ++++++ .../polaris/core/entity/CatalogEntity.java | 8 ++ .../DefaultServiceIdentityRegistry.java | 3 +- .../ServiceIdentityRegistry.java | 3 +- .../ServiceIdentityRegistryFactory.java | 2 +- .../admintool/config/QuarkusProducers.java | 7 ++ .../quarkus/config/QuarkusProducers.java | 4 +- ...PolarisCredentialManagerConfiguration.java | 34 +++++++ ...sServiceIdentityRegistryConfiguration.java | 2 +- .../CatalogEntityConnectionConfigMutator.java | 2 +- .../iceberg/IcebergCatalogAdapter.java | 5 + .../iceberg/IcebergCatalogHandler.java | 14 ++- .../context/DefaultCallContextResolver.java | 9 +- ...efaultPolarisCredentialManagerFactory.java | 57 ++++++++++++ ...DefaultServiceIdentityRegistryFactory.java | 6 +- ...tomicOperationMetaStoreManagerFactory.java | 9 +- ...nMemoryPolarisMetaStoreManagerFactory.java | 11 ++- 23 files changed, 413 insertions(+), 25 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManager.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManagerFactory.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/ConnectionCredentialProperty.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/PolarisConnectionCredsVendor.java rename polaris-core/src/main/java/org/apache/polaris/core/identity/{ => registry}/DefaultServiceIdentityRegistry.java (95%) rename polaris-core/src/main/java/org/apache/polaris/core/identity/{ => registry}/ServiceIdentityRegistry.java (92%) rename polaris-core/src/main/java/org/apache/polaris/core/identity/{ => registry}/ServiceIdentityRegistryFactory.java (95%) create mode 100644 quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusPolarisCredentialManagerConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/credentials/DefaultPolarisCredentialManagerFactory.java diff --git a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index 492f61668d..67a64d26b8 100644 --- a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -26,6 +26,7 @@ import java.nio.file.Path; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; @@ -44,14 +45,16 @@ public class EclipseLinkPolarisMetaStoreManagerFactory @Inject EclipseLinkConfiguration eclipseLinkConfiguration; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject EntityMutationEngine entityMutationEngine; protected EclipseLinkPolarisMetaStoreManagerFactory() { - this(null); + this(null, null); } @Inject - protected EclipseLinkPolarisMetaStoreManagerFactory(PolarisDiagnostics diagnostics) { - super(diagnostics); + protected EclipseLinkPolarisMetaStoreManagerFactory( + PolarisDiagnostics diagnostics, EntityMutationEngine entityMutationEngine) { + super(diagnostics, entityMutationEngine); } @Override diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index aa7eff7858..081b9f29f7 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -38,6 +38,7 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; @@ -74,6 +75,7 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject Instance dataSource; @Inject RelationalJdbcConfiguration relationalJdbcConfiguration; + @Inject EntityMutationEngine entityMutationEngine; protected JdbcMetaStoreManagerFactory() {} @@ -158,7 +160,8 @@ public Map purgeRealms(Iterable realms) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); BasePersistence session = getOrCreateSessionSupplier(realmContext).get(); - PolarisCallContext callContext = new PolarisCallContext(realmContext, session, diagServices); + PolarisCallContext callContext = + new PolarisCallContext(realmContext, session, diagServices, entityMutationEngine); BaseResult result = metaStoreManager.purge(callContext); results.put(realm, result); @@ -229,7 +232,8 @@ private PrincipalSecretsResult bootstrapServiceAndCreatePolarisPrincipalForRealm new PolarisCallContext( realmContext, sessionSupplierMap.get(realmContext.getRealmIdentifier()).get(), - diagServices); + diagServices, + entityMutationEngine); if (CallContext.getCurrentContext() == null) { CallContext.setCurrentContext(polarisContext); } @@ -280,7 +284,8 @@ private void checkPolarisServiceBootstrappedForRealm( new PolarisCallContext( realmContext, sessionSupplierMap.get(realmContext.getRealmIdentifier()).get(), - diagServices); + diagServices, + entityMutationEngine); if (CallContext.getCurrentContext() == null) { CallContext.setCurrentContext(polarisContext); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java new file mode 100644 index 0000000000..50c567b1a1 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.credentials; + +import java.util.EnumMap; +import java.util.Optional; +import org.apache.polaris.core.connection.AuthenticationParametersDpo; +import org.apache.polaris.core.connection.SigV4AuthenticationParametersDpo; +import org.apache.polaris.core.credentials.connection.ConnectionCredentialProperty; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistry; +import org.apache.polaris.core.identity.resolved.ResolvedAwsIamServiceIdentity; +import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; + +public class DefaultPolarisCredentialManager implements PolarisCredentialManager { + private final ServiceIdentityRegistry serviceIdentityRegistry; + + public DefaultPolarisCredentialManager(ServiceIdentityRegistry serviceIdentityRegistry) { + this.serviceIdentityRegistry = serviceIdentityRegistry; + } + + @Override + public EnumMap getConnectionCredentials( + ServiceIdentityInfoDpo serviceIdentity, + AuthenticationParametersDpo authenticationParameters) { + EnumMap credentialMap = + new EnumMap<>(ConnectionCredentialProperty.class); + ResolvedServiceIdentity resolvedServiceIdentity = + serviceIdentityRegistry.resolveServiceIdentity(serviceIdentity); + if (resolvedServiceIdentity == null) { + return credentialMap; + } + + switch (serviceIdentity.getIdentityType()) { + case AWS_IAM: + ResolvedAwsIamServiceIdentity resolvedAwsIamServiceIdentity = + (ResolvedAwsIamServiceIdentity) resolvedServiceIdentity; + SigV4AuthenticationParametersDpo sigV4AuthenticationParameters = + (SigV4AuthenticationParametersDpo) authenticationParameters; + StsClient stsClient = resolvedAwsIamServiceIdentity.stsClientSupplier().get(); + AssumeRoleResponse response = + stsClient.assumeRole( + AssumeRoleRequest.builder() + .roleArn(sigV4AuthenticationParameters.getRoleArn()) + .roleSessionName( + Optional.ofNullable(sigV4AuthenticationParameters.getRoleSessionName()) + .orElse("polaris")) + .externalId(sigV4AuthenticationParameters.getExternalId()) + .build()); + credentialMap.put( + ConnectionCredentialProperty.AWS_ACCESS_KEY_ID, response.credentials().accessKeyId()); + credentialMap.put( + ConnectionCredentialProperty.AWS_SECRET_ACCESS_KEY, + response.credentials().secretAccessKey()); + credentialMap.put( + ConnectionCredentialProperty.AWS_SESSION_TOKEN, response.credentials().sessionToken()); + Optional.ofNullable(response.credentials().expiration()) + .ifPresent( + i -> { + credentialMap.put( + ConnectionCredentialProperty.EXPIRATION_TIME, + String.valueOf(i.toEpochMilli())); + }); + break; + default: + LoggerFactory.getLogger(DefaultPolarisCredentialManager.class) + .warn("Unsupported service identity type: {}", serviceIdentity.getIdentityType()); + return credentialMap; + } + return credentialMap; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManager.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManager.java new file mode 100644 index 0000000000..568b1b7675 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManager.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.credentials; + +import java.util.EnumMap; +import org.apache.polaris.core.connection.AuthenticationParametersDpo; +import org.apache.polaris.core.credentials.connection.ConnectionCredentialProperty; +import org.apache.polaris.core.credentials.connection.PolarisConnectionCredsVendor; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; + +public interface PolarisCredentialManager extends PolarisConnectionCredsVendor { + @Override + EnumMap getConnectionCredentials( + ServiceIdentityInfoDpo serviceIdentity, AuthenticationParametersDpo authenticationParameters); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManagerFactory.java new file mode 100644 index 0000000000..eeacbdd017 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManagerFactory.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.credentials; + +import org.apache.polaris.core.context.RealmContext; + +/** + * Factory for creating {@link PolarisCredentialManager} instances. + * + *

Each {@link PolarisCredentialManager} instance is associated with a {@link RealmContext} and + * is responsible for managing the credentials for the user in that realm. + */ +public interface PolarisCredentialManagerFactory { + PolarisCredentialManager getOrCreatePolarisCredentialManager(RealmContext realmContext); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/ConnectionCredentialProperty.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/ConnectionCredentialProperty.java new file mode 100644 index 0000000000..fe99a9e739 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/ConnectionCredentialProperty.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.credentials.connection; + +import org.apache.iceberg.aws.AwsProperties; + +public enum ConnectionCredentialProperty { + AWS_ACCESS_KEY_ID(String.class, AwsProperties.REST_ACCESS_KEY_ID, "the aws access key id"), + AWS_SECRET_ACCESS_KEY( + String.class, AwsProperties.REST_SECRET_ACCESS_KEY, "the aws access key secret"), + AWS_SESSION_TOKEN(String.class, AwsProperties.REST_SESSION_TOKEN, "the aws scoped access token"), + EXPIRATION_TIME( + Long.class, "expiration-time", "the expiration time for the access token, in milliseconds"); + + private final Class valueType; + private final String propertyName; + private final String description; + private final boolean isCredential; + + ConnectionCredentialProperty(Class valueType, String propertyName, String description) { + this(valueType, propertyName, description, true); + } + + ConnectionCredentialProperty( + Class valueType, String propertyName, String description, boolean isCredential) { + this.valueType = valueType; + this.propertyName = propertyName; + this.description = description; + this.isCredential = isCredential; + } + + public String getPropertyName() { + return propertyName; + } + + public boolean isCredential() { + return isCredential; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/PolarisConnectionCredsVendor.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/PolarisConnectionCredsVendor.java new file mode 100644 index 0000000000..5ee892a7cf --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/PolarisConnectionCredsVendor.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.credentials.connection; + +import java.util.EnumMap; +import org.apache.polaris.core.connection.AuthenticationParametersDpo; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; + +public interface PolarisConnectionCredsVendor { + EnumMap getConnectionCredentials( + ServiceIdentityInfoDpo serviceIdentity, AuthenticationParametersDpo authenticationParameters); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index a995883304..2e736fec25 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -330,6 +330,14 @@ public Builder setConnectionConfigInfoDpoWithSecrets( return this; } + public Builder setConnectionConfigInfoDpo( + @Nonnull ConnectionConfigInfoDpo connectionConfigInfoDpo) { + internalProperties.put( + PolarisEntityConstants.getConnectionConfigInfoPropertyName(), + connectionConfigInfoDpo.serialize()); + return this; + } + @Override public CatalogEntity build() { return new CatalogEntity(buildBase()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/DefaultServiceIdentityRegistry.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java similarity index 95% rename from polaris-core/src/main/java/org/apache/polaris/core/identity/DefaultServiceIdentityRegistry.java rename to polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java index 23ab19ddaf..d89aba55a7 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/DefaultServiceIdentityRegistry.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java @@ -17,11 +17,12 @@ * under the License. */ -package org.apache.polaris.core.identity; +package org.apache.polaris.core.identity.registry; import java.util.EnumMap; import java.util.Map; import java.util.stream.Collectors; +import org.apache.polaris.core.identity.ServiceIdentityType; import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistry.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistry.java similarity index 92% rename from polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistry.java rename to polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistry.java index 4177521d3c..e81fcb2007 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistry.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistry.java @@ -17,8 +17,9 @@ * under the License. */ -package org.apache.polaris.core.identity; +package org.apache.polaris.core.identity.registry; +import org.apache.polaris.core.identity.ServiceIdentityType; import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistryFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistryFactory.java similarity index 95% rename from polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistryFactory.java rename to polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistryFactory.java index 7b20e42c5e..385f6b5d77 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/ServiceIdentityRegistryFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistryFactory.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.polaris.core.identity; +package org.apache.polaris.core.identity.registry; import org.apache.polaris.core.context.RealmContext; diff --git a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java index 07ad023629..59d3619a13 100644 --- a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java +++ b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java @@ -28,6 +28,7 @@ import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -71,6 +72,12 @@ PolarisStorageIntegration getStorageIntegrationForConfig( }; } + @Produces + public EntityMutationEngine entityMutationEngine() { + // An entity mutation engine is not required when running the admin tool. + return entity -> entity; + } + @Produces public PolarisConfigurationStore configurationStore() { // A configuration store is not required when running the admin tool. diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 1e4f9d780b..2763bb6597 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -44,9 +44,9 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.credentials.PolarisCredentialManager; import org.apache.polaris.core.credentials.PolarisCredentialManagerFactory; -import org.apache.polaris.core.identity.ServiceIdentityRegistry; -import org.apache.polaris.core.identity.ServiceIdentityRegistryFactory; import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistry; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistryFactory; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusPolarisCredentialManagerConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusPolarisCredentialManagerConfiguration.java new file mode 100644 index 0000000000..7aeafb8915 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusPolarisCredentialManagerConfiguration.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.identity; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.credential-manager") +public interface QuarkusPolarisCredentialManagerConfiguration { + + /** + * The type of the PolarisCredentialManagerFactory to use. This is the {@link + * org.apache.polaris.core.credentials.PolarisCredentialManagerFactory} identifier. + */ + String type(); +} diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityRegistryConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityRegistryConfiguration.java index 38038a3ec5..d689f9eb47 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityRegistryConfiguration.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusServiceIdentityRegistryConfiguration.java @@ -21,7 +21,7 @@ import io.quarkus.runtime.annotations.StaticInitSafe; import io.smallrye.config.ConfigMapping; -import org.apache.polaris.core.identity.ServiceIdentityRegistryFactory; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistryFactory; @StaticInitSafe @ConfigMapping(prefix = "polaris.service-identity.registry") diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java index b859ef221e..1f2e8ad8e7 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java @@ -26,8 +26,8 @@ import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.identity.ServiceIdentityRegistry; import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistry; import org.apache.polaris.service.identity.mutation.EntityMutator; @RequestScoped diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index c28fd491a6..2f887d0215 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -62,6 +62,7 @@ import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.credentials.PolarisCredentialManager; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -137,6 +138,7 @@ public class IcebergCatalogAdapter private final PolarisEntityManager entityManager; private final PolarisMetaStoreManager metaStoreManager; private final UserSecretsManager userSecretsManager; + private final PolarisCredentialManager credentialManager; private final PolarisAuthorizer polarisAuthorizer; private final CatalogPrefixParser prefixParser; private final ReservedProperties reservedProperties; @@ -150,6 +152,7 @@ public IcebergCatalogAdapter( PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, UserSecretsManager userSecretsManager, + PolarisCredentialManager credentialManager, PolarisAuthorizer polarisAuthorizer, CatalogPrefixParser prefixParser, ReservedProperties reservedProperties, @@ -160,6 +163,7 @@ public IcebergCatalogAdapter( this.entityManager = entityManager; this.metaStoreManager = metaStoreManager; this.userSecretsManager = userSecretsManager; + this.credentialManager = credentialManager; this.polarisAuthorizer = polarisAuthorizer; this.prefixParser = prefixParser; this.reservedProperties = reservedProperties; @@ -198,6 +202,7 @@ private IcebergCatalogHandler newHandlerWrapper( entityManager, metaStoreManager, userSecretsManager, + credentialManager, securityContext, catalogFactory, catalogName, diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index c06e9d98d4..405272e36f 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -82,6 +82,7 @@ import org.apache.polaris.core.connection.hadoop.HadoopConnectionConfigInfoDpo; import org.apache.polaris.core.connection.iceberg.IcebergRestConnectionConfigInfoDpo; import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.credentials.PolarisCredentialManager; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; @@ -125,6 +126,7 @@ public class IcebergCatalogHandler extends CatalogHandler implements AutoCloseab private final PolarisMetaStoreManager metaStoreManager; private final UserSecretsManager userSecretsManager; + private final PolarisCredentialManager credentialManager; private final CallContextCatalogFactory catalogFactory; private final ReservedProperties reservedProperties; private final CatalogHandlerUtils catalogHandlerUtils; @@ -143,6 +145,7 @@ public IcebergCatalogHandler( PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, UserSecretsManager userSecretsManager, + PolarisCredentialManager credentialManager, SecurityContext securityContext, CallContextCatalogFactory catalogFactory, String catalogName, @@ -152,6 +155,7 @@ public IcebergCatalogHandler( super(callContext, entityManager, securityContext, catalogName, authorizer); this.metaStoreManager = metaStoreManager; this.userSecretsManager = userSecretsManager; + this.credentialManager = credentialManager; this.catalogFactory = catalogFactory; this.reservedProperties = reservedProperties; this.catalogHandlerUtils = catalogHandlerUtils; @@ -199,6 +203,10 @@ private UserSecretsManager getUserSecretsManager() { return userSecretsManager; } + private PolarisCredentialManager getPolarisCredentialManager() { + return credentialManager; + } + @Override protected void initializeCatalog() { CatalogEntity resolvedCatalogEntity = @@ -229,13 +237,15 @@ protected void initializeCatalog() { .build()); federatedCatalog.initialize( ((IcebergRestConnectionConfigInfoDpo) connectionConfigInfoDpo).getRemoteCatalogName(), - connectionConfigInfoDpo.asIcebergCatalogProperties(getUserSecretsManager())); + connectionConfigInfoDpo.asIcebergCatalogProperties( + getUserSecretsManager(), getPolarisCredentialManager())); break; case HADOOP: federatedCatalog = new HadoopCatalog(); federatedCatalog.initialize( ((HadoopConnectionConfigInfoDpo) connectionConfigInfoDpo).getWarehouse(), - connectionConfigInfoDpo.asIcebergCatalogProperties(getUserSecretsManager())); + connectionConfigInfoDpo.asIcebergCatalogProperties( + getUserSecretsManager(), getPolarisCredentialManager())); break; default: throw new UnsupportedOperationException( diff --git a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java index 62b113daea..934d69d5af 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java @@ -28,6 +28,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.slf4j.Logger; @@ -47,6 +48,7 @@ public class DefaultCallContextResolver implements CallContextResolver { @Inject MetaStoreManagerFactory metaStoreManagerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisDiagnostics diagnostics; + @Inject EntityMutationEngine entityMutationEngine; @Inject Clock clock; @Override @@ -67,6 +69,11 @@ public CallContext resolveCallContext( BasePersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); return new PolarisCallContext( - realmContext, metaStoreSession, diagnostics, configurationStore, clock); + realmContext, + metaStoreSession, + diagnostics, + configurationStore, + entityMutationEngine, + clock); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/credentials/DefaultPolarisCredentialManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/credentials/DefaultPolarisCredentialManagerFactory.java new file mode 100644 index 0000000000..6a914476e5 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/credentials/DefaultPolarisCredentialManagerFactory.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.credentials; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.credentials.DefaultPolarisCredentialManager; +import org.apache.polaris.core.credentials.PolarisCredentialManager; +import org.apache.polaris.core.credentials.PolarisCredentialManagerFactory; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistry; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistryFactory; + +@ApplicationScoped +@Identifier("default") +public class DefaultPolarisCredentialManagerFactory implements PolarisCredentialManagerFactory { + private final Map cachedCredentialManagers = + new ConcurrentHashMap<>(); + + ServiceIdentityRegistryFactory serviceIdentityRegistryFactory; + + @Inject + public DefaultPolarisCredentialManagerFactory( + ServiceIdentityRegistryFactory serviceIdentityRegistryFactory) { + this.serviceIdentityRegistryFactory = serviceIdentityRegistryFactory; + } + + @Override + public PolarisCredentialManager getOrCreatePolarisCredentialManager(RealmContext realmContext) { + ServiceIdentityRegistry serviceIdentityRegistry = + serviceIdentityRegistryFactory.getOrCreateServiceIdentityRegistry(realmContext); + + return cachedCredentialManagers.computeIfAbsent( + realmContext.getRealmIdentifier(), + key -> new DefaultPolarisCredentialManager(serviceIdentityRegistry)); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java b/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java index 821a47fefc..b9b1279cc8 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java @@ -26,10 +26,10 @@ import java.util.Optional; import java.util.stream.Collectors; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.identity.DefaultServiceIdentityRegistry; -import org.apache.polaris.core.identity.ServiceIdentityRegistry; -import org.apache.polaris.core.identity.ServiceIdentityRegistryFactory; import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.identity.registry.DefaultServiceIdentityRegistry; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistry; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistryFactory; import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; import org.apache.polaris.core.secrets.ServiceSecretReference; diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java index 3f0b1a3552..ffdb73b266 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java @@ -22,6 +22,7 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -36,13 +37,15 @@ public class InMemoryAtomicOperationMetaStoreManagerFactory extends InMemoryPolarisMetaStoreManagerFactory { public InMemoryAtomicOperationMetaStoreManagerFactory() { - super(null, null); + super(null, null, null); } @Inject public InMemoryAtomicOperationMetaStoreManagerFactory( - PolarisStorageIntegrationProvider storageIntegration, PolarisDiagnostics diagnostics) { - super(storageIntegration, diagnostics); + PolarisStorageIntegrationProvider storageIntegration, + PolarisDiagnostics diagnostics, + EntityMutationEngine entityMutationEngine) { + super(storageIntegration, diagnostics, entityMutationEngine); } @Override diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 895e5b51b1..135bc2dc05 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -30,6 +30,7 @@ import java.util.function.Supplier; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; @@ -45,17 +46,21 @@ public class InMemoryPolarisMetaStoreManagerFactory extends LocalPolarisMetaStoreManagerFactory { private final PolarisStorageIntegrationProvider storageIntegration; + private final EntityMutationEngine entityMutationEngine; private final Set bootstrappedRealms = new HashSet<>(); public InMemoryPolarisMetaStoreManagerFactory() { - this(null, null); + this(null, null, null); } @Inject public InMemoryPolarisMetaStoreManagerFactory( - PolarisStorageIntegrationProvider storageIntegration, PolarisDiagnostics diagnostics) { - super(diagnostics); + PolarisStorageIntegrationProvider storageIntegration, + PolarisDiagnostics diagnostics, + EntityMutationEngine entityMutationEngine) { + super(diagnostics, entityMutationEngine); this.storageIntegration = storageIntegration; + this.entityMutationEngine = entityMutationEngine; } @Override From 5af3d4987b405e21312e8989f396a92d628672d5 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Tue, 27 May 2025 17:21:13 -0700 Subject: [PATCH 06/15] Fix compilation errors --- ...olarisEclipseLinkMetaStoreManagerTest.java | 2 ++ ...anagerWithJdbcBasePersistenceImplTest.java | 2 ++ ...apAtomicOperationMetaStoreManagerTest.java | 2 ++ .../PolarisTreeMapMetaStoreManagerTest.java | 2 ++ .../core/persistence/ResolverTest.java | 6 +++++- .../cache/InMemoryEntityCacheTest.java | 9 ++++++++- .../InMemoryStorageIntegrationTest.java | 2 ++ .../cache/StorageCredentialCacheTest.java | 6 +++++- .../quarkus/admin/ManagementServiceTest.java | 4 +++- .../quarkus/admin/PolarisAuthzTestBase.java | 10 ++++++++++ .../quarkus/auth/JWTRSAKeyPairTest.java | 2 +- .../auth/JWTSymmetricKeyGeneratorTest.java | 2 +- .../IcebergCatalogHandlerAuthzTest.java | 3 +++ .../quarkus/catalog/IcebergCatalogTest.java | 3 +++ .../catalog/IcebergCatalogViewTest.java | 3 +++ .../PolarisGenericTableCatalogTest.java | 3 +++ .../quarkus/catalog/PolicyCatalogTest.java | 3 +++ .../config/DefaultConfigurationStoreTest.java | 2 ++ .../quarkus/entity/CatalogEntityTest.java | 6 +++++- .../task/BatchFileCleanupTaskHandlerTest.java | 11 +++++++--- .../ManifestFileCleanupTaskHandlerTest.java | 14 +++++++++---- .../task/TableCleanupTaskHandlerTest.java | 3 +++ .../test/PolarisIntegrationTestFixture.java | 1 + .../test/PolarisIntegrationTestHelper.java | 2 ++ .../service/catalog/io/FileIOFactoryTest.java | 1 + .../service/task/TaskExecutorImplTest.java | 6 +++++- .../apache/polaris/service/TestServices.java | 20 ++++++++++++++++++- 27 files changed, 114 insertions(+), 16 deletions(-) diff --git a/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java b/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java index 55607981e9..0acb6a75bb 100644 --- a/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java +++ b/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java @@ -39,6 +39,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; import org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl; @@ -96,6 +97,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { session, diagServices, new PolarisConfigurationStore() {}, + new NoOpEntityMutationEngine(), timeSource.withZone(ZoneId.systemDefault()))); } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index 1df5d6d5f0..705c19baf9 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -29,6 +29,7 @@ import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; @@ -71,6 +72,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { basePersistence, diagServices, new PolarisConfigurationStore() {}, + new NoOpEntityMutationEngine(), timeSource.withZone(ZoneId.systemDefault()))); } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapAtomicOperationMetaStoreManagerTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapAtomicOperationMetaStoreManagerTest.java index f89615cf10..c001a5a3d0 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapAtomicOperationMetaStoreManagerTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapAtomicOperationMetaStoreManagerTest.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.persistence.transactional.TreeMapMetaStore; import org.apache.polaris.core.persistence.transactional.TreeMapTransactionalPersistenceImpl; import org.mockito.Mockito; @@ -41,6 +42,7 @@ public PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS), diagServices, new PolarisConfigurationStore() {}, + new NoOpEntityMutationEngine(), timeSource.withZone(ZoneId.systemDefault())); return new PolarisTestMetaStoreManager(new AtomicOperationMetaStoreManager(), callCtx); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java index 49a2bfcc00..97a8b5e665 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl; import org.apache.polaris.core.persistence.transactional.TreeMapMetaStore; import org.apache.polaris.core.persistence.transactional.TreeMapTransactionalPersistenceImpl; @@ -41,6 +42,7 @@ public PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS), diagServices, new PolarisConfigurationStore() {}, + new NoOpEntityMutationEngine(), timeSource.withZone(ZoneId.systemDefault())); return new PolarisTestMetaStoreManager(new TransactionalMetaStoreManagerImpl(), callCtx); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java index d72a199ae7..06477144c6 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java @@ -22,6 +22,8 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl; import org.apache.polaris.core.persistence.transactional.TreeMapMetaStore; import org.apache.polaris.core.persistence.transactional.TreeMapTransactionalPersistenceImpl; @@ -40,7 +42,9 @@ protected PolarisCallContext callCtx() { TreeMapMetaStore store = new TreeMapMetaStore(diagServices); TreeMapTransactionalPersistenceImpl metaStore = new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS); - callCtx = new PolarisCallContext(() -> "testRealm", metaStore, diagServices); + EntityMutationEngine entityMutationEngine = new NoOpEntityMutationEngine(); + callCtx = + new PolarisCallContext(() -> "testRealm", metaStore, diagServices, entityMutationEngine); } return callCtx; } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java index 1d0564be9e..ba697340ea 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java @@ -33,6 +33,8 @@ import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; @@ -65,6 +67,9 @@ public class InMemoryEntityCacheTest { // the meta store manager private final PolarisMetaStoreManager metaStoreManager; + // the entity mutation engine + private final EntityMutationEngine entityMutationEngine; + /** * Initialize and create the test metadata * @@ -91,7 +96,9 @@ public InMemoryEntityCacheTest() { diagServices = new PolarisDefaultDiagServiceImpl(); store = new TreeMapMetaStore(diagServices); metaStore = new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS); - callCtx = new PolarisCallContext(() -> "testRealm", metaStore, diagServices); + entityMutationEngine = new NoOpEntityMutationEngine(); + callCtx = + new PolarisCallContext(() -> "testRealm", metaStore, diagServices, entityMutationEngine); metaStoreManager = new TransactionalMetaStoreManagerImpl(); // bootstrap the mata store with our test schema diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java index a4e58860de..7f5008301d 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java @@ -31,6 +31,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -107,6 +108,7 @@ public void testValidateAccessToLocationsWithWildcard() { return (T) config.get(configName); } }, + new NoOpEntityMutationEngine(), Clock.systemUTC()); CallContext.setCurrentContext(polarisCallContext); Map> result = diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index b1d1789dac..93799e687b 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -37,6 +37,8 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; import org.apache.polaris.core.persistence.dao.entity.BaseResult; @@ -70,7 +72,9 @@ public StorageCredentialCacheTest() { // to interact with the metastore TransactionalPersistence metaStore = new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS); - callCtx = new PolarisCallContext(() -> "testRealm", metaStore, diagServices); + EntityMutationEngine entityMutationEngine = new NoOpEntityMutationEngine(); + callCtx = + new PolarisCallContext(() -> "testRealm", metaStore, diagServices, entityMutationEngine); metaStoreManager = Mockito.mock(PolarisMetaStoreManager.class); storageCredentialCache = new StorageCredentialCache(); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java index 47128c86d1..0fd8c0323e 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java @@ -75,6 +75,7 @@ public void setup() { .get(), fakeServices.polarisDiagnostics(), fakeServices.configurationStore(), + fakeServices.entityMutationEngine(), Mockito.mock(Clock.class)); CallContext.setCurrentContext(polarisCallContext); services = @@ -188,7 +189,8 @@ private PolarisCallContext setupCallContext(PolarisMetaStoreManager metaStoreMan return new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - services.polarisDiagnostics()); + services.polarisDiagnostics(), + services.entityMutationEngine()); } private PolarisAdminService setupPolarisAdminService( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 4e8748de55..3721819835 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -59,6 +59,8 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.credentials.PolarisCredentialManager; +import org.apache.polaris.core.credentials.PolarisCredentialManagerFactory; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.CatalogRoleEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -68,6 +70,8 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -193,6 +197,7 @@ public Map getConfigOverrides() { @Inject protected RealmEntityManagerFactory realmEntityManagerFactory; @Inject protected CallContextCatalogFactory callContextCatalogFactory; @Inject protected UserSecretsManagerFactory userSecretsManagerFactory; + @Inject protected PolarisCredentialManagerFactory credentialManagerFactory; @Inject protected PolarisDiagnostics diagServices; @Inject protected Clock clock; @Inject protected FileIOFactory fileIOFactory; @@ -206,7 +211,9 @@ public Map getConfigOverrides() { protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; protected PolarisMetaStoreManager metaStoreManager; + protected EntityMutationEngine entityMutationEngine; protected UserSecretsManager userSecretsManager; + protected PolarisCredentialManager credentialManager; protected TransactionalPersistence metaStoreSession; protected PolarisBaseEntity catalogEntity; protected PrincipalEntity principalEntity; @@ -232,6 +239,8 @@ public void before(TestInfo testInfo) { QuarkusMock.installMockForType(realmContext, RealmContext.class); metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); + credentialManager = credentialManagerFactory.getOrCreatePolarisCredentialManager(realmContext); + entityMutationEngine = new NoOpEntityMutationEngine(); polarisAuthorizer = new PolarisAuthorizerImpl(configurationStore); @@ -241,6 +250,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityMutationEngine, clock); this.entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java index 46d2950b91..b633050ef8 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java @@ -61,7 +61,7 @@ public void testSuccessfulTokenGeneration() throws Exception { final String scope = "PRINCIPAL_ROLE:TEST"; PolarisCallContext polarisCallContext = - new PolarisCallContext(null, null, null, configurationStore, null); + new PolarisCallContext(null, null, null, configurationStore, null, null); PolarisMetaStoreManager metastoreManager = Mockito.mock(PolarisMetaStoreManager.class); String mainSecret = "client-secret"; PolarisPrincipalSecrets principalSecrets = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java index 6b51f6eda3..ff8a734584 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java @@ -45,7 +45,7 @@ public class JWTSymmetricKeyGeneratorTest { /** Sanity test to verify that we can generate a token */ @Test public void testJWTSymmetricKeyGenerator() { - PolarisCallContext polarisCallContext = new PolarisCallContext(null, null, null, null, null); + PolarisCallContext polarisCallContext = new PolarisCallContext(null, null, null, null, null, null); PolarisMetaStoreManager metastoreManager = Mockito.mock(PolarisMetaStoreManager.class); String mainSecret = "test_secret"; String clientId = "test_client_id"; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 5c488809d1..fd2bef81f1 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -100,6 +100,7 @@ private IcebergCatalogHandler newWrapper( entityManager, metaStoreManager, userSecretsManager, + credentialManager, securityContext(authenticatedPrincipal, activatedPrincipalRoles), factory, catalogName, @@ -241,6 +242,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { entityManager, metaStoreManager, userSecretsManager, + credentialManager, securityContext(authenticatedPrincipal, Set.of(PRINCIPAL_ROLE1, PRINCIPAL_ROLE2)), callContextCatalogFactory, CATALOG_NAME, @@ -276,6 +278,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { entityManager, metaStoreManager, userSecretsManager, + credentialManager, securityContext(authenticatedPrincipal1, Set.of(PRINCIPAL_ROLE1, PRINCIPAL_ROLE2)), callContextCatalogFactory, CATALOG_NAME, diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 2b4c4205c9..4a5d155881 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -89,6 +89,7 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.TaskEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -210,6 +211,7 @@ public Map getConfigOverrides() { @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisDiagnostics diagServices; @Inject PolarisEventListener polarisEventListener; + @Inject EntityMutationEngine entityMutationEngine; private IcebergCatalog catalog; private String realmName; @@ -253,6 +255,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityMutationEngine, Clock.systemDefaultZone()); entityManager = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 0f06a02250..172ddeb827 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -55,6 +55,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -132,6 +133,7 @@ public Map getConfigOverrides() { @Inject PolarisConfigurationStore configurationStore; @Inject PolarisDiagnostics diagServices; @Inject PolarisEventListener polarisEventListener; + @Inject EntityMutationEngine entityMutationEngine; private IcebergCatalog catalog; @@ -174,6 +176,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityMutationEngine, Clock.systemDefaultZone()); PolarisEntityManager entityManager = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java index 2aba1773b9..d53ca65c91 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java @@ -58,6 +58,7 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.table.GenericTableEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -128,6 +129,7 @@ public Map getConfigOverrides() { @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject PolarisDiagnostics diagServices; + @Inject EntityMutationEngine entityMutationEngine; private PolarisGenericTableCatalog genericTableCatalog; private IcebergCatalog icebergCatalog; @@ -173,6 +175,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityMutationEngine, Clock.systemDefaultZone()); entityManager = new PolarisEntityManager( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java index 93ee060723..8e8c672605 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java @@ -66,6 +66,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -158,6 +159,7 @@ public Map getConfigOverrides() { @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject PolarisDiagnostics diagServices; + @Inject EntityMutationEngine entityMutationEngine; private PolicyCatalog policyCatalog; private IcebergCatalog icebergCatalog; @@ -199,6 +201,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityMutationEngine, Clock.systemDefaultZone()); entityManager = new PolarisEntityManager( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java index 3da4bd2950..46c3511e18 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java @@ -29,6 +29,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.service.config.DefaultConfigurationStore; import org.apache.polaris.service.config.FeaturesConfiguration; import org.assertj.core.api.Assertions; @@ -72,6 +73,7 @@ public Map getConfigOverrides() { @Inject PolarisConfigurationStore configurationStore; @Inject FeaturesConfiguration featuresConfiguration; + @Inject EntityMutationEngine entityMutationEngine; @BeforeEach public void before(TestInfo testInfo) { diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java index 0a6dea8fed..44c9b4ab72 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java @@ -31,6 +31,8 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.assertj.core.api.Assertions; @@ -45,11 +47,13 @@ public class CatalogEntityTest { public static void setup() { MetaStoreManagerFactory metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); RealmContext realmContext = () -> "realm"; + EntityMutationEngine entityMutationEngine = new NoOpEntityMutationEngine(); PolarisCallContext polarisCallContext = new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(() -> "realm").get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityMutationEngine); CallContext.setCurrentContext(polarisCallContext); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/BatchFileCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/BatchFileCleanupTaskHandlerTest.java index 5a4de3109a..498566bc4a 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/BatchFileCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/BatchFileCleanupTaskHandlerTest.java @@ -51,6 +51,7 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisTaskConstants; import org.apache.polaris.core.entity.TaskEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -63,6 +64,7 @@ @QuarkusTest public class BatchFileCleanupTaskHandlerTest { @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject EntityMutationEngine entityMutationEngine; private final RealmContext realmContext = () -> "realmName"; private TaskFileIOSupplier buildTaskFileIOSupplier(FileIO fileIO) { @@ -94,7 +96,8 @@ public void testMetadataFileCleanup() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityMutationEngine); FileIO fileIO = new InMemoryFileIO() { @Override @@ -207,7 +210,8 @@ public void testMetadataFileCleanupIfFileNotExist() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityMutationEngine); CallContext.setCurrentContext(polarisCallContext); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); @@ -252,7 +256,8 @@ public void testCleanupWithRetries() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityMutationEngine); CallContext.setCurrentContext(polarisCallContext); Map retryCounter = new HashMap<>(); FileIO fileIO = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java index e6ee6724e3..989f92ac0e 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java @@ -49,6 +49,7 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisTaskConstants; import org.apache.polaris.core.entity.TaskEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -61,6 +62,7 @@ @QuarkusTest class ManifestFileCleanupTaskHandlerTest { @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject EntityMutationEngine entityMutationEngine; private final RealmContext realmContext = () -> "realmName"; @@ -93,7 +95,8 @@ public void testCleanupFileNotExists() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityMutationEngine); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); @@ -123,7 +126,8 @@ public void testCleanupFileManifestExistsDataFilesDontExist() throws IOException new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityMutationEngine); CallContext.setCurrentContext(polarisCallContext); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); @@ -152,7 +156,8 @@ public void testCleanupFiles() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityMutationEngine); CallContext.setCurrentContext(polarisCallContext); FileIO fileIO = new InMemoryFileIO() { @@ -198,7 +203,8 @@ public void testCleanupFilesWithRetries() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityMutationEngine); CallContext.setCurrentContext(polarisCallContext); Map retryCounter = new HashMap<>(); FileIO fileIO = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java index 5c968f7e88..3c7e92704e 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java @@ -53,6 +53,7 @@ import org.apache.polaris.core.entity.PolarisTaskConstants; import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; @@ -75,6 +76,7 @@ class TableCleanupTaskHandlerTest { @Inject MetaStoreManagerFactory metaStoreManagerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisDiagnostics diagServices; + @Inject EntityMutationEngine entityMutationEngine; private CallContext callContext; @@ -107,6 +109,7 @@ void setup() { metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityMutationEngine, Clock.systemDefaultZone()); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java index e9268befb7..f17058f2e2 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java @@ -118,6 +118,7 @@ private PolarisPrincipalSecrets fetchAdminSecrets() { metaStoreSession, helper.diagServices, helper.configurationStore, + helper.entityMutationEngine, helper.clock); try { PolarisMetaStoreManager metaStoreManager = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java index 7149291543..100b2e1132 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java @@ -24,6 +24,7 @@ import java.time.Clock; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.context.RealmContextResolver; import org.junit.jupiter.api.TestInfo; @@ -36,6 +37,7 @@ public class PolarisIntegrationTestHelper { @Inject ObjectMapper objectMapper; @Inject PolarisDiagnostics diagServices; @Inject PolarisConfigurationStore configurationStore; + @Inject EntityMutationEngine entityMutationEngine; @Inject Clock clock; public PolarisIntegrationTestFixture createFixture(TestEnvironment testEnv, TestInfo testInfo) { diff --git a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java index ee47c14f2d..38207c74af 100644 --- a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java @@ -141,6 +141,7 @@ FileIO loadFileIOInternal( testServices.metaStoreManagerFactory().getOrCreateSessionSupplier(realmContext).get(), testServices.polarisDiagnostics(), testServices.configurationStore(), + testServices.entityMutationEngine(), Clock.systemUTC()); } diff --git a/service/common/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java b/service/common/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java index c67f7f172a..6bee318518 100644 --- a/service/common/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java @@ -50,7 +50,11 @@ void testEventsAreEmitted() { BasePersistence bp = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); PolarisCallContext polarisCallCtx = - new PolarisCallContext(realmContext, bp, testServices.polarisDiagnostics()); + new PolarisCallContext( + realmContext, + bp, + testServices.polarisDiagnostics(), + testServices.entityMutationEngine()); // This task doesn't have a type so it won't be handle-able by a real handler. We register a // test TaskHandler below that can handle any task. diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index d3622126fb..31550c8edd 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -37,8 +37,13 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.credentials.PolarisCredentialManager; +import org.apache.polaris.core.credentials.PolarisCredentialManagerFactory; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.mutation.NoOpEntityMutationEngine; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistryFactory; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; @@ -59,8 +64,10 @@ import org.apache.polaris.service.config.ReservedProperties; import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; import org.apache.polaris.service.context.catalog.PolarisCallContextCatalogFactory; +import org.apache.polaris.service.credentials.DefaultPolarisCredentialManagerFactory; import org.apache.polaris.service.events.PolarisEventListener; import org.apache.polaris.service.events.TestPolarisEventListener; +import org.apache.polaris.service.identity.DefaultServiceIdentityRegistryFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.secrets.UnsafeInMemorySecretsManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; @@ -75,6 +82,7 @@ public record TestServices( IcebergRestConfigurationApi restConfigurationApi, PolarisConfigurationStore configurationStore, PolarisDiagnostics polarisDiagnostics, + EntityMutationEngine entityMutationEngine, RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, RealmContext realmContext, @@ -152,13 +160,18 @@ public TestServices build() { () -> stsClient, Optional.empty(), () -> GoogleCredentials.create(new AccessToken(GCP_ACCESS_TOKEN, new Date()))); + EntityMutationEngine entityMutationEngine = new NoOpEntityMutationEngine(); InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory( - storageIntegrationProvider, polarisDiagnostics); + storageIntegrationProvider, polarisDiagnostics, entityMutationEngine); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(metaStoreManagerFactory) {}; UserSecretsManagerFactory userSecretsManagerFactory = new UnsafeInMemorySecretsManagerFactory(); + ServiceIdentityRegistryFactory serviceIdentityRegistryFactory = + new DefaultServiceIdentityRegistryFactory(); + PolarisCredentialManagerFactory credentialManagerFactory = + new DefaultPolarisCredentialManagerFactory(serviceIdentityRegistryFactory); BasePersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); @@ -168,6 +181,7 @@ public TestServices build() { metaStoreSession, polarisDiagnostics, configurationStore, + entityMutationEngine, Clock.systemUTC()); PolarisEntityManager entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); @@ -175,6 +189,8 @@ public TestServices build() { metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); UserSecretsManager userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); + PolarisCredentialManager credentialManager = + credentialManagerFactory.getOrCreatePolarisCredentialManager(realmContext); FileIOFactory fileIOFactory = fileIOFactorySupplier.apply( @@ -205,6 +221,7 @@ public TestServices build() { entityManager, metaStoreManager, userSecretsManager, + credentialManager, authorizer, new DefaultCatalogPrefixParser(), reservedProperties, @@ -264,6 +281,7 @@ public String getAuthenticationScheme() { restConfigurationApi, configurationStore, polarisDiagnostics, + entityMutationEngine, realmEntityManagerFactory, metaStoreManagerFactory, realmContext, From 8bfa2a1f814d23a6efc8341aa41571f9bb8c70ab Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Tue, 27 May 2025 18:06:37 -0700 Subject: [PATCH 07/15] Add DPO tests for the new added DPOs --- .../ConnectionConfigInfoDpoTest.java | 63 +++++++++++++++++++ .../identity/ResolvedServiceIdentityTest.java | 52 +++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/identity/ResolvedServiceIdentityTest.java diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java index 3aeed0b05f..d12448cc81 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java @@ -131,4 +131,67 @@ void testBearerAuthenticationParameters() throws JsonProcessingException { objectMapper.readValue(expectedApiModelJson, ConnectionConfigInfo.class), connectionConfigInfoApiModel); } + + @Test + void testSigV4AuthenticationParameters() throws JsonProcessingException { + // Test deserialization and reserialization of the persistence JSON. + String json = + "" + + "{" + + " \"connectionTypeCode\": 1," + + " \"uri\": \"https://glue.us-west-2.amazonaws.com/iceberg\"," + + " \"remoteCatalogName\": \"123456789012\"," + + " \"authenticationParameters\": {" + + " \"authenticationTypeCode\": 3," + + " \"roleArn\": \"arn:aws:iam::123456789012:role/glue-catalog-role\"," + + " \"roleSessionName\": \"polaris-catalog-federation\"," + + " \"externalId\": \"external-id\"," + + " \"signingRegion\": \"us-west-2\"," + + " \"signingName\": \"glue\"" + + " }," + + " \"serviceIdentity\": {" + + " \"identityTypeCode\": 1," + + " \"iamArn\": \"arn:aws:iam::123456789012:user/polaris-iam-user\"," + + " \"identityInfoReference\": {" + + " \"urn\": \"urn:polaris-service-secret:default-identity-registry:my-realm:AWS_IAM\"," + + " \"referencePayload\": {" + + " \"key\": \"value\"" + + " }" + + " }" + + " }" + + "}"; + + ConnectionConfigInfoDpo connectionConfigInfoDpo = + ConnectionConfigInfoDpo.deserialize(polarisDiagnostics, json); + Assertions.assertNotNull(connectionConfigInfoDpo); + JsonNode tree1 = objectMapper.readTree(json); + JsonNode tree2 = objectMapper.readTree(connectionConfigInfoDpo.serialize()); + Assertions.assertEquals(tree1, tree2); + + // Test conversion into API model JSON. + ConnectionConfigInfo connectionConfigInfoApiModel = + connectionConfigInfoDpo.asConnectionConfigInfoModel(); + String expectedApiModelJson = + "" + + "{" + + " \"connectionType\": \"ICEBERG_REST\"," + + " \"uri\": \"https://glue.us-west-2.amazonaws.com/iceberg\"," + + " \"remoteCatalogName\": \"123456789012\"," + + " \"authenticationParameters\": {" + + " \"authenticationType\": \"SIGV4\"," + + " \"roleArn\": \"arn:aws:iam::123456789012:role/glue-catalog-role\"," + + " \"roleSessionName\": \"polaris-catalog-federation\"," + + " \"externalId\": \"external-id\"," + + " \"signingRegion\": \"us-west-2\"," + + " \"signingName\": \"glue\"" + + " }," + + " \"serviceIdentity\": {" + + " \"identityType\": \"AWS_IAM\"," + + " \"iamArn\": \"arn:aws:iam::123456789012:user/polaris-iam-user\"" + + " }" + + "}"; + Assertions.assertEquals( + objectMapper.readValue(expectedApiModelJson, ConnectionConfigInfo.class), + connectionConfigInfoApiModel); + } } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/identity/ResolvedServiceIdentityTest.java b/polaris-core/src/test/java/org/apache/polaris/core/identity/ResolvedServiceIdentityTest.java new file mode 100644 index 0000000000..a67f929f6a --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/identity/ResolvedServiceIdentityTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.identity; + +import java.util.Map; +import org.apache.polaris.core.identity.dpo.AwsIamServiceIdentityInfoDpo; +import org.apache.polaris.core.identity.resolved.ResolvedAwsIamServiceIdentity; +import org.apache.polaris.core.secrets.ServiceSecretReference; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ResolvedServiceIdentityTest { + + @Test + void testResolvedAwsIamServiceIdentity() { + ResolvedAwsIamServiceIdentity identity = + new ResolvedAwsIamServiceIdentity( + "arn:aws:iam::123456789012:user/polaris-iam-user", + "access-key-id", + "secret-access-key", + "session-token"); + AwsIamServiceIdentityInfoDpo dpo = + (AwsIamServiceIdentityInfoDpo) identity.asServiceIdentityInfoDpo(); + Assertions.assertThat(dpo.getIdentityType()).isEqualTo(ServiceIdentityType.AWS_IAM); + Assertions.assertThat(dpo.getIamArn()) + .isEqualTo("arn:aws:iam::123456789012:user/polaris-iam-user"); + + ServiceSecretReference identityInfoReference = + new ServiceSecretReference( + "urn:polaris-service-secret:defualt-identity-registry:my-realm:aws-iam", Map.of()); + identity.setIdentityInfoReference(identityInfoReference); + dpo = (AwsIamServiceIdentityInfoDpo) identity.asServiceIdentityInfoDpo(); + Assertions.assertThat(dpo.getIdentityInfoReference()).isEqualTo(identityInfoReference); + } +} From 9fe926480415a23cb2ed82bde406aa6d107fa022 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Wed, 28 May 2025 12:58:24 -0700 Subject: [PATCH 08/15] Add some tests for ServiceIdentityRegistry and fix some bugs --- .../DefaultServiceIdentityRegistry.java | 6 + .../ResolvedAwsIamServiceIdentity.java | 12 +- .../core/secrets/ServiceSecretReference.java | 2 +- ...PolarisCredentialManagerConfiguration.java | 2 +- .../DefaultServiceIdentityRegistryTest.java | 169 ++++++++++++++++++ ...DefaultServiceIdentityRegistryFactory.java | 11 +- 6 files changed, 198 insertions(+), 4 deletions(-) rename quarkus/service/src/main/java/org/apache/polaris/service/quarkus/{identity => credential}/QuarkusPolarisCredentialManagerConfiguration.java (95%) create mode 100644 quarkus/service/src/test/java/org/apache/polaris/service/quarkus/identity/DefaultServiceIdentityRegistryTest.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java index d89aba55a7..a30e2d55b0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.identity.registry; +import com.google.common.annotations.VisibleForTesting; import java.util.EnumMap; import java.util.Map; import java.util.stream.Collectors; @@ -61,4 +62,9 @@ public ResolvedServiceIdentity resolveServiceIdentity( serviceIdentityInfo.getIdentityInfoReference().getUrn()); return resolvedServiceIdentity; } + + @VisibleForTesting + public EnumMap getResolvedServiceIdentities() { + return resolvedServiceIdentities; + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java index e3e52a974a..ffec4a89e1 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java @@ -24,6 +24,7 @@ import org.apache.polaris.core.identity.ServiceIdentityType; import org.apache.polaris.core.identity.dpo.AwsIamServiceIdentityInfoDpo; import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +import org.apache.polaris.core.secrets.ServiceSecretReference; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -38,7 +39,16 @@ public class ResolvedAwsIamServiceIdentity extends ResolvedServiceIdentity { public ResolvedAwsIamServiceIdentity( String iamArn, String accessKeyId, String secretAccessKey, String sessionToken) { - super(ServiceIdentityType.AWS_IAM); + this(null, iamArn, accessKeyId, secretAccessKey, sessionToken); + } + + public ResolvedAwsIamServiceIdentity( + ServiceSecretReference serviceSecretReference, + String iamArn, + String accessKeyId, + String secretAccessKey, + String sessionToken) { + super(ServiceIdentityType.AWS_IAM, serviceSecretReference); this.iamArn = iamArn; this.accessKeyId = accessKeyId; this.secretAccessKey = secretAccessKey; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/ServiceSecretReference.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/ServiceSecretReference.java index 2ab37feea0..a81eb3fdf6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/ServiceSecretReference.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/ServiceSecretReference.java @@ -47,7 +47,7 @@ public ServiceSecretReference( super(urn, referencePayload); // TODO: Add better/standardized parsing and validation of URN syntax Preconditions.checkArgument( - urn.startsWith("urn:polaris-service-secret:") && urn.split(":").length >= 5, + urn.startsWith("urn:polaris-service-secret:") && urn.split(":").length >= 4, "Invalid secret URN '%s'; must be of the form " + "'urn:polaris-service-secret::'", urn); diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusPolarisCredentialManagerConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credential/QuarkusPolarisCredentialManagerConfiguration.java similarity index 95% rename from quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusPolarisCredentialManagerConfiguration.java rename to quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credential/QuarkusPolarisCredentialManagerConfiguration.java index 7aeafb8915..fd5f6f94fa 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/QuarkusPolarisCredentialManagerConfiguration.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credential/QuarkusPolarisCredentialManagerConfiguration.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.polaris.service.quarkus.identity; +package org.apache.polaris.service.quarkus.credential; import io.quarkus.runtime.annotations.StaticInitSafe; import io.smallrye.config.ConfigMapping; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/identity/DefaultServiceIdentityRegistryTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/identity/DefaultServiceIdentityRegistryTest.java new file mode 100644 index 0000000000..218961cfb3 --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/identity/DefaultServiceIdentityRegistryTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.identity; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import java.util.EnumMap; +import java.util.Map; +import java.util.Optional; +import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.identity.registry.DefaultServiceIdentityRegistry; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistryFactory; +import org.apache.polaris.core.identity.resolved.ResolvedAwsIamServiceIdentity; +import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; +import org.apache.polaris.core.secrets.ServiceSecretReference; +import org.apache.polaris.service.identity.ServiceIdentityConfiguration; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +@QuarkusTest +@TestProfile(DefaultServiceIdentityRegistryTest.Profile.class) +public class DefaultServiceIdentityRegistryTest { + private static final String DEFAULT_REALM_KEY = ServiceIdentityConfiguration.DEFAULT_REALM_KEY; + private static final String MY_REALM_KEY = "my-realm"; + + @Inject QuarkusServiceIdentityConfiguration serviceIdentityConfiguration; + @Inject ServiceIdentityRegistryFactory serviceIdentityRegistryFactory; + + public static class Profile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "quarkus.identity-registry.type", + "default", + "polaris.service-identity.aws-iam.iam-arn", + "arn:aws:iam::123456789012:user/polaris-default-iam-user", + "polaris.service-identity.my-realm.aws-iam.iam-arn", + "arn:aws:iam::123456789012:user/polaris-iam-user", + "polaris.service-identity.my-realm.aws-iam.access-key-id", + "access-key-id", + "polaris.service-identity.my-realm.aws-iam.secret-access-key", + "secret-access-key", + "polaris.service-identity.my-realm.aws-iam.session-token", + "session-token"); + } + } + + @Test + void testServiceIdentityConfiguration() { + // Ensure that the service identity configuration is loaded correctly + Assertions.assertThat(serviceIdentityConfiguration.realms()).isNotNull(); + Assertions.assertThat(serviceIdentityConfiguration.realms()) + .containsKey(ServiceIdentityConfiguration.DEFAULT_REALM_KEY) + .containsKey(MY_REALM_KEY) + .size() + .isEqualTo(2); + + // Check the default realm configuration + QuarkusRealmServiceIdentityConfiguration defaultConfig = + serviceIdentityConfiguration.forRealm(DEFAULT_REALM_KEY); + Assertions.assertThat(defaultConfig.awsIamServiceIdentity().isPresent()).isTrue(); + Assertions.assertThat(defaultConfig.awsIamServiceIdentity().get().iamArn()) + .isEqualTo("arn:aws:iam::123456789012:user/polaris-default-iam-user"); + Assertions.assertThat(defaultConfig.awsIamServiceIdentity().get().accessKeyId()).isEmpty(); + Assertions.assertThat(defaultConfig.awsIamServiceIdentity().get().secretAccessKey()).isEmpty(); + Assertions.assertThat(defaultConfig.awsIamServiceIdentity().get().sessionToken()).isEmpty(); + + // Check the my-realm configuration + QuarkusRealmServiceIdentityConfiguration myRealmConfig = + serviceIdentityConfiguration.forRealm(MY_REALM_KEY); + Assertions.assertThat(myRealmConfig.awsIamServiceIdentity().isPresent()).isTrue(); + Assertions.assertThat(myRealmConfig.awsIamServiceIdentity().get().iamArn()) + .isEqualTo("arn:aws:iam::123456789012:user/polaris-iam-user"); + Assertions.assertThat(myRealmConfig.awsIamServiceIdentity().get().accessKeyId()) + .isEqualTo(Optional.of("access-key-id")); + Assertions.assertThat(myRealmConfig.awsIamServiceIdentity().get().secretAccessKey()) + .isEqualTo(Optional.of("secret-access-key")); + Assertions.assertThat(myRealmConfig.awsIamServiceIdentity().get().sessionToken()) + .isEqualTo(Optional.of("session-token")); + + // Check the unexisting realm configuration + QuarkusRealmServiceIdentityConfiguration otherConfig = + serviceIdentityConfiguration.forRealm("other-realm"); + Assertions.assertThat(otherConfig.awsIamServiceIdentity().isPresent()).isTrue(); + Assertions.assertThat(otherConfig.awsIamServiceIdentity().get().iamArn()) + .isEqualTo("arn:aws:iam::123456789012:user/polaris-default-iam-user"); + Assertions.assertThat(otherConfig.awsIamServiceIdentity().get().accessKeyId()).isEmpty(); + Assertions.assertThat(otherConfig.awsIamServiceIdentity().get().secretAccessKey()).isEmpty(); + Assertions.assertThat(otherConfig.awsIamServiceIdentity().get().sessionToken()).isEmpty(); + } + + @Test + void testRealmServiceIdentityConfigToResolvedServiceIdentity() { + // Check the default realm + DefaultServiceIdentityRegistry defaultRegistry = + (DefaultServiceIdentityRegistry) + serviceIdentityRegistryFactory.getOrCreateServiceIdentityRegistry( + () -> DEFAULT_REALM_KEY); + EnumMap resolvedIdentities = + defaultRegistry.getResolvedServiceIdentities(); + + Assertions.assertThat(resolvedIdentities) + .containsKey(ServiceIdentityType.AWS_IAM) + .size() + .isEqualTo(1); + ResolvedAwsIamServiceIdentity resolvedAwsIamServiceIdentity = + (ResolvedAwsIamServiceIdentity) resolvedIdentities.get(ServiceIdentityType.AWS_IAM); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getIamArn()) + .isEqualTo("arn:aws:iam::123456789012:user/polaris-default-iam-user"); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getIdentityInfoReference()) + .isEqualTo( + new ServiceSecretReference( + "urn:polaris-service-secret:default-identity-registry:system:default:AWS_IAM", + Map.of())); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getAccessKeyId()).isNull(); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getSecretAccessKey()).isNull(); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getSessionToken()).isNull(); + + // Check the my-realm + DefaultServiceIdentityRegistry myRealmRegistry = + (DefaultServiceIdentityRegistry) + serviceIdentityRegistryFactory.getOrCreateServiceIdentityRegistry(() -> MY_REALM_KEY); + resolvedIdentities = myRealmRegistry.getResolvedServiceIdentities(); + + Assertions.assertThat(resolvedIdentities) + .containsKey(ServiceIdentityType.AWS_IAM) + .size() + .isEqualTo(1); + resolvedAwsIamServiceIdentity = + (ResolvedAwsIamServiceIdentity) resolvedIdentities.get(ServiceIdentityType.AWS_IAM); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getIamArn()) + .isEqualTo("arn:aws:iam::123456789012:user/polaris-iam-user"); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getIdentityInfoReference()) + .isEqualTo( + new ServiceSecretReference( + "urn:polaris-service-secret:default-identity-registry:my-realm:AWS_IAM", Map.of())); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getAccessKeyId()) + .isEqualTo("access-key-id"); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getSecretAccessKey()) + .isEqualTo("secret-access-key"); + Assertions.assertThat(resolvedAwsIamServiceIdentity.getSessionToken()) + .isEqualTo("session-token"); + + // Check the other realm + DefaultServiceIdentityRegistry otherRegistry = + (DefaultServiceIdentityRegistry) + serviceIdentityRegistryFactory.getOrCreateServiceIdentityRegistry(() -> "other-realm"); + Assertions.assertThat(otherRegistry).isEqualTo(defaultRegistry); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java b/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java index b9b1279cc8..f2df25ccc4 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.identity; +import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -37,6 +38,7 @@ @Identifier("default") public class DefaultServiceIdentityRegistryFactory implements ServiceIdentityRegistryFactory { private static final String DEFAULT_REALM_KEY = ServiceIdentityConfiguration.DEFAULT_REALM_KEY; + private static final String DEFAULT_REALM_NSS = "system:default"; private static final String IDENTITY_INFO_REFERENCE_URN_FORMAT = "urn:polaris-service-secret:default-identity-registry:%s:%s"; @@ -108,6 +110,11 @@ public ServiceIdentityRegistry getOrCreateServiceIdentityRegistry(RealmContext r return getServiceIdentityRegistryForRealm(realmContext); } + @VisibleForTesting + public Map getRealmServiceIdentityRegistries() { + return realmServiceIdentityRegistries; + } + protected DefaultServiceIdentityRegistry getServiceIdentityRegistryForRealm( RealmContext realmContext) { return getServiceIdentityRegistryForRealm(realmContext.getRealmIdentifier()); @@ -123,6 +130,8 @@ private ServiceSecretReference buildIdentityInfoReference( String realm, ServiceIdentityType type) { // urn:polaris-service-secret:default-identity-registry:: return new ServiceSecretReference( - IDENTITY_INFO_REFERENCE_URN_FORMAT.formatted(realm, type.name()), Map.of()); + IDENTITY_INFO_REFERENCE_URN_FORMAT.formatted( + realm.equals(DEFAULT_REALM_KEY) ? DEFAULT_REALM_NSS : realm, type.name()), + Map.of()); } } From f70b46754940affc0de43e0cad535b0c2ed21343 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Thu, 29 May 2025 09:37:00 +0000 Subject: [PATCH 09/15] Retrieve connection credentials in SigV4 DPO --- .../SigV4AuthenticationParametersDpo.java | 13 +++++++++---- .../iceberg/IcebergRestConnectionConfigInfoDpo.java | 9 +++++---- .../service/quarkus/config/QuarkusProducers.java | 3 ++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java index 7250f49606..e10f1f213f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java @@ -23,12 +23,14 @@ import com.google.common.collect.ImmutableMap; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import java.util.EnumMap; import java.util.Map; import org.apache.iceberg.aws.AwsProperties; import org.apache.iceberg.rest.auth.AuthProperties; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.SigV4AuthenticationParameters; import org.apache.polaris.core.credentials.PolarisCredentialManager; +import org.apache.polaris.core.credentials.connection.ConnectionCredentialProperty; import org.apache.polaris.core.secrets.UserSecretsManager; /** @@ -96,10 +98,13 @@ public Map asIcebergCatalogProperties( if (getSigningName() != null) { builder.put(AwsProperties.REST_SIGNING_NAME, getSigningName()); } - // TODO: Add a connection credential provider to get the tmp aws credentials for SigV4 auth - // builder.put(AwsProperties.REST_ACCESS_KEY_ID, "access_key_id"); - // builder.put(AwsProperties.REST_SECRET_ACCESS_KEY, "secret_access_key"); - // builder.put(AwsProperties.REST_SESSION_TOKEN, "session_token"); + + EnumMap connectionCredentialProperties = + credentialManager.getConnectionCredentials(null, this); + if (connectionCredentialProperties != null) { + connectionCredentialProperties.forEach( + (key, value) -> builder.put(key.getPropertyName(), value)); + } return builder.build(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java index ed51e65698..eb3bcb791a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java @@ -71,10 +71,11 @@ public String getRemoteCatalogName() { } properties.putAll( getAuthenticationParameters() - .asIcebergCatalogProperties(secretsManager, credentialManager)); - credentialManager - .getConnectionCredentials(getServiceIdentity(), getAuthenticationParameters()) - .forEach((key, value) -> properties.put(key.getPropertyName(), value)); + .asIcebergCatalogProperties( + secretsManager, + (serviceIdentity, authenticationParameters) -> + credentialManager.getConnectionCredentials( + getServiceIdentity(), authenticationParameters))); return properties; } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 2763bb6597..87d6901a84 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -74,6 +74,7 @@ import org.apache.polaris.service.quarkus.catalog.io.QuarkusFileIOConfiguration; import org.apache.polaris.service.quarkus.context.QuarkusRealmContextConfiguration; import org.apache.polaris.service.quarkus.context.RealmContextFilter; +import org.apache.polaris.service.quarkus.credential.QuarkusPolarisCredentialManagerConfiguration; import org.apache.polaris.service.quarkus.events.QuarkusPolarisEventListenerConfiguration; import org.apache.polaris.service.quarkus.identity.QuarkusRealmServiceIdentityConfiguration; import org.apache.polaris.service.quarkus.identity.QuarkusServiceIdentityConfiguration; @@ -199,7 +200,7 @@ public ServiceIdentityRegistryFactory serviceIdentityRegistryFactory( @Produces public PolarisCredentialManagerFactory credentialManagerFactory( - QuarkusServiceIdentityRegistryConfiguration config, + QuarkusPolarisCredentialManagerConfiguration config, @Any Instance credentialManagerFactories) { return credentialManagerFactories.select(Identifier.Literal.of(config.type())).get(); } From 5072aa3dff12987d075183fdd3828cf032bc1868 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Mon, 2 Jun 2025 22:13:13 -0700 Subject: [PATCH 10/15] Support configurable entity mutation pipeline via Quarkus config --- .../src/main/resources/application.properties | 2 ++ .../quarkus/config/QuarkusProducers.java | 8 +++++ .../CatalogEntityConnectionConfigMutator.java | 9 ++---- .../identity/mutation/NoOpEntityMutator.java | 9 ++---- .../QuarkusEntityMutationConfiguration.java | 31 ++++++++++++++++++ .../mutation/EntityMutationConfiguration.java | 27 ++++++++++++++++ .../mutation/EntityMutationEngineImpl.java | 32 ++++++++++++++----- .../identity/mutation/EntityMutator.java | 2 -- 8 files changed, 96 insertions(+), 24 deletions(-) create mode 100644 quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/QuarkusEntityMutationConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationConfiguration.java diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index f83338f4e1..7a9e51d93d 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -188,6 +188,8 @@ polaris.oidc.principal-roles-mapper.type=default # polaris.storage.gcp.token=token # polaris.storage.gcp.lifespan=PT1H +polaris.entity-mutation.mutators=no-op,catalog-connection-config + # Polaris Service Identity Config polaris.service-identity.registry.type=default polaris.service-identity.aws-iam.iam-arn=arn:aws:iam::123456789012:user/polaris-iam-user diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 87d6901a84..557c9ccd9a 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -68,6 +68,7 @@ import org.apache.polaris.service.context.RealmContextResolver; import org.apache.polaris.service.events.PolarisEventListener; import org.apache.polaris.service.identity.ServiceIdentityConfiguration; +import org.apache.polaris.service.identity.mutation.EntityMutationConfiguration; import org.apache.polaris.service.quarkus.auth.QuarkusAuthenticationConfiguration; import org.apache.polaris.service.quarkus.auth.QuarkusAuthenticationRealmConfiguration; import org.apache.polaris.service.quarkus.auth.external.tenant.OidcTenantResolver; @@ -79,6 +80,7 @@ import org.apache.polaris.service.quarkus.identity.QuarkusRealmServiceIdentityConfiguration; import org.apache.polaris.service.quarkus.identity.QuarkusServiceIdentityConfiguration; import org.apache.polaris.service.quarkus.identity.QuarkusServiceIdentityRegistryConfiguration; +import org.apache.polaris.service.quarkus.identity.mutation.QuarkusEntityMutationConfiguration; import org.apache.polaris.service.quarkus.persistence.QuarkusPersistenceConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusRateLimiterFilterConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusTokenBucketConfiguration; @@ -198,6 +200,12 @@ public ServiceIdentityRegistryFactory serviceIdentityRegistryFactory( return serviceIdentityRegistryFactories.select(Identifier.Literal.of(config.type())).get(); } +// @Produces +// public EntityMutationConfiguration entityMutationConfiguration( +// QuarkusEntityMutationConfiguration config) { +// return config; +// } + @Produces public PolarisCredentialManagerFactory credentialManagerFactory( QuarkusPolarisCredentialManagerConfiguration config, diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java index 1f2e8ad8e7..b35add0a80 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java @@ -19,6 +19,7 @@ package org.apache.polaris.service.quarkus.identity.mutation; +import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import org.apache.polaris.core.connection.AuthenticationParametersDpo; @@ -31,10 +32,9 @@ import org.apache.polaris.service.identity.mutation.EntityMutator; @RequestScoped +@Identifier("catalog-connection-config") public class CatalogEntityConnectionConfigMutator implements EntityMutator { - private static final String MUTATOR_ID = "catalog-connection-config"; - ServiceIdentityRegistry serviceIdentityRegistry; @Inject @@ -42,11 +42,6 @@ public class CatalogEntityConnectionConfigMutator implements EntityMutator { this.serviceIdentityRegistry = serviceIdentityRegistry; } - @Override - public String id() { - return MUTATOR_ID; - } - @Override public PolarisBaseEntity apply(PolarisBaseEntity entity) { if (!(entity instanceof CatalogEntity catalogEntity) || !catalogEntity.isPassthroughFacade()) { diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java index 13f7783320..b317b62391 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java @@ -19,19 +19,14 @@ package org.apache.polaris.service.quarkus.identity.mutation; +import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.ApplicationScoped; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.service.identity.mutation.EntityMutator; @ApplicationScoped +@Identifier("no-op") public class NoOpEntityMutator implements EntityMutator { - private static final String MUTATOR_ID = "no-op"; - - @Override - public String id() { - return MUTATOR_ID; - } - @Override public PolarisBaseEntity apply(PolarisBaseEntity entity) { return entity; diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/QuarkusEntityMutationConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/QuarkusEntityMutationConfiguration.java new file mode 100644 index 0000000000..04f58b4dce --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/QuarkusEntityMutationConfiguration.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.identity.mutation; + +import io.smallrye.config.ConfigMapping; +import java.util.List; +import java.util.Optional; +import org.apache.polaris.service.identity.mutation.EntityMutationConfiguration; + +@ConfigMapping(prefix = "polaris.entity-mutation") +public interface QuarkusEntityMutationConfiguration extends EntityMutationConfiguration { + @Override + Optional> mutators(); +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationConfiguration.java new file mode 100644 index 0000000000..9301cadcce --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationConfiguration.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.identity.mutation; + +import java.util.List; +import java.util.Optional; + +public interface EntityMutationConfiguration { + Optional> mutators(); +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java index 86cbd20a13..74c4396aad 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java @@ -19,34 +19,50 @@ package org.apache.polaris.service.identity.mutation; +import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Any; import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; -import java.util.Map; -import java.util.stream.Collectors; +import java.util.List; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.identity.mutation.EntityMutationEngine; @ApplicationScoped public class EntityMutationEngineImpl implements EntityMutationEngine { - private final Instance mutatorInstance; + private final EntityMutationConfiguration config; + private final Instance mutatorInstances; @Inject - public EntityMutationEngineImpl(Instance mutatorInstance) { - this.mutatorInstance = mutatorInstance; + public EntityMutationEngineImpl( + EntityMutationConfiguration config, @Any Instance mutatorInstances) { + this.config = config; + this.mutatorInstances = mutatorInstances; } @Override public PolarisBaseEntity applyMutations(PolarisBaseEntity entity) { PolarisBaseEntity result = entity; - Map mutatorMap = - mutatorInstance.stream().collect(Collectors.toMap(EntityMutator::id, m -> m)); + List orderedMutators = + // config.map(EntityMutationConfiguration::mutators).orElse(List.of()).stream() + config.mutators().orElse(List.of()).stream() + .map( + id -> { + Instance matched = + mutatorInstances.select(Identifier.Literal.of(id)); + if (matched.isResolvable()) { + return matched.get(); + } else { + throw new IllegalStateException("No EntityMutator found for ID: " + id); + } + }) + .toList(); // Apply all mutators to the entity // TODO: Add a way to control the order of mutators - for (EntityMutator mutator : mutatorMap.values()) { + for (EntityMutator mutator : orderedMutators) { result = mutator.apply(result); } diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java index 7254770d86..261515999e 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java @@ -22,7 +22,5 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; public interface EntityMutator { - String id(); - PolarisBaseEntity apply(PolarisBaseEntity entity); } From 18f80f77883ebe75c7ec5365ffa4f282474cad24 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Mon, 2 Jun 2025 22:17:15 -0700 Subject: [PATCH 11/15] Rename package name from credential to credentials --- .../service/quarkus/config/QuarkusProducers.java | 10 +--------- .../QuarkusPolarisCredentialManagerConfiguration.java | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) rename quarkus/service/src/main/java/org/apache/polaris/service/quarkus/{credential => credentials}/QuarkusPolarisCredentialManagerConfiguration.java (95%) diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 557c9ccd9a..bf18bbc601 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -68,19 +68,17 @@ import org.apache.polaris.service.context.RealmContextResolver; import org.apache.polaris.service.events.PolarisEventListener; import org.apache.polaris.service.identity.ServiceIdentityConfiguration; -import org.apache.polaris.service.identity.mutation.EntityMutationConfiguration; import org.apache.polaris.service.quarkus.auth.QuarkusAuthenticationConfiguration; import org.apache.polaris.service.quarkus.auth.QuarkusAuthenticationRealmConfiguration; import org.apache.polaris.service.quarkus.auth.external.tenant.OidcTenantResolver; import org.apache.polaris.service.quarkus.catalog.io.QuarkusFileIOConfiguration; import org.apache.polaris.service.quarkus.context.QuarkusRealmContextConfiguration; import org.apache.polaris.service.quarkus.context.RealmContextFilter; -import org.apache.polaris.service.quarkus.credential.QuarkusPolarisCredentialManagerConfiguration; +import org.apache.polaris.service.quarkus.credentials.QuarkusPolarisCredentialManagerConfiguration; import org.apache.polaris.service.quarkus.events.QuarkusPolarisEventListenerConfiguration; import org.apache.polaris.service.quarkus.identity.QuarkusRealmServiceIdentityConfiguration; import org.apache.polaris.service.quarkus.identity.QuarkusServiceIdentityConfiguration; import org.apache.polaris.service.quarkus.identity.QuarkusServiceIdentityRegistryConfiguration; -import org.apache.polaris.service.quarkus.identity.mutation.QuarkusEntityMutationConfiguration; import org.apache.polaris.service.quarkus.persistence.QuarkusPersistenceConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusRateLimiterFilterConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusTokenBucketConfiguration; @@ -200,12 +198,6 @@ public ServiceIdentityRegistryFactory serviceIdentityRegistryFactory( return serviceIdentityRegistryFactories.select(Identifier.Literal.of(config.type())).get(); } -// @Produces -// public EntityMutationConfiguration entityMutationConfiguration( -// QuarkusEntityMutationConfiguration config) { -// return config; -// } - @Produces public PolarisCredentialManagerFactory credentialManagerFactory( QuarkusPolarisCredentialManagerConfiguration config, diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credential/QuarkusPolarisCredentialManagerConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credentials/QuarkusPolarisCredentialManagerConfiguration.java similarity index 95% rename from quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credential/QuarkusPolarisCredentialManagerConfiguration.java rename to quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credentials/QuarkusPolarisCredentialManagerConfiguration.java index fd5f6f94fa..77e58c50ee 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credential/QuarkusPolarisCredentialManagerConfiguration.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credentials/QuarkusPolarisCredentialManagerConfiguration.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.polaris.service.quarkus.credential; +package org.apache.polaris.service.quarkus.credentials; import io.quarkus.runtime.annotations.StaticInitSafe; import io.smallrye.config.ConfigMapping; From 407791ca4982c14f7b787a5c3b506dfdac007211 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Tue, 3 Jun 2025 16:02:15 -0700 Subject: [PATCH 12/15] Add a test for DefaultPolarisCredentialManager --- .../DefaultPolarisCredentialManager.java | 10 +- .../DefaultPolarisCredentialManagerTest.java | 139 ++++++++++++++++++ .../mutation/EntityMutationConfiguration.java | 2 +- 3 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 quarkus/service/src/test/java/org/apache/polaris/service/quarkus/credentials/DefaultPolarisCredentialManagerTest.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java index 50c567b1a1..ef4e203625 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.credentials; +import com.google.common.annotations.VisibleForTesting; import java.util.EnumMap; import java.util.Optional; import org.apache.polaris.core.connection.AuthenticationParametersDpo; @@ -29,6 +30,7 @@ import org.apache.polaris.core.identity.resolved.ResolvedAwsIamServiceIdentity; import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.annotations.NotNull; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; @@ -58,7 +60,7 @@ public EnumMap getConnectionCredentials( (ResolvedAwsIamServiceIdentity) resolvedServiceIdentity; SigV4AuthenticationParametersDpo sigV4AuthenticationParameters = (SigV4AuthenticationParametersDpo) authenticationParameters; - StsClient stsClient = resolvedAwsIamServiceIdentity.stsClientSupplier().get(); + StsClient stsClient = getStsClient(resolvedAwsIamServiceIdentity); AssumeRoleResponse response = stsClient.assumeRole( AssumeRoleRequest.builder() @@ -90,4 +92,10 @@ public EnumMap getConnectionCredentials( } return credentialMap; } + + @VisibleForTesting + public StsClient getStsClient( + @NotNull ResolvedAwsIamServiceIdentity resolvedAwsIamServiceIdentity) { + return resolvedAwsIamServiceIdentity.stsClientSupplier().get(); + } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/credentials/DefaultPolarisCredentialManagerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/credentials/DefaultPolarisCredentialManagerTest.java new file mode 100644 index 0000000000..7808614798 --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/credentials/DefaultPolarisCredentialManagerTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.credentials; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import io.quarkus.test.InjectMock; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import java.time.Instant; +import java.util.EnumMap; +import java.util.Map; +import org.apache.polaris.core.connection.SigV4AuthenticationParametersDpo; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.credentials.DefaultPolarisCredentialManager; +import org.apache.polaris.core.credentials.connection.ConnectionCredentialProperty; +import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +import org.apache.polaris.core.identity.registry.ServiceIdentityRegistry; +import org.apache.polaris.core.identity.resolved.ResolvedAwsIamServiceIdentity; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; +import software.amazon.awssdk.services.sts.model.Credentials; + +@QuarkusTest +@TestProfile(DefaultPolarisCredentialManagerTest.Profile.class) +public class DefaultPolarisCredentialManagerTest { + + @InjectMock RealmContext realmContext; + + @Inject QuarkusPolarisCredentialManagerConfiguration configuration; + @Inject ServiceIdentityRegistry serviceIdentityRegistry; + + DefaultPolarisCredentialManager credentialManager; + + public static class Profile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "quarkus.identity-registry.type", + "default", + "polaris.service-identity.my-realm.aws-iam.iam-arn", + "arn:aws:iam::123456789012:user/polaris-iam-user", + "polaris.service-identity.my-realm.aws-iam.access-key-id", + "access-key-id", + "polaris.service-identity.my-realm.aws-iam.secret-access-key", + "secret-access-key", + "polaris.credential-manager.type", + "default"); + } + } + + @BeforeEach + void setup() { + // Mock the realm context to return a specific realm + when(realmContext.getRealmIdentifier()).thenReturn("my-realm"); + + credentialManager = Mockito.spy(new DefaultPolarisCredentialManager(serviceIdentityRegistry)); + doAnswer( + invocation -> { + // Capture the identity here + ResolvedAwsIamServiceIdentity identity = invocation.getArgument(0); + + StsClient mockStsClient = mock(StsClient.class); + when(mockStsClient.assumeRole(Mockito.any(AssumeRoleRequest.class))) + .thenAnswer( + stsInvocation -> { + // Validate identity at the time assumeRole is called + if (!"access-key-id".equals(identity.getAccessKeyId()) + || !"secret-access-key".equals(identity.getSecretAccessKey())) { + throw new IllegalArgumentException("Invalid credentials on assumeRole"); + } + + // Return mocked credentials + Credentials tmpSessionCredentials = + Credentials.builder() + .accessKeyId("tmp-access-key-id") + .secretAccessKey("tmp-secret-access-key") + .sessionToken("tmp-session-token") + .expiration(Instant.now().plusSeconds(3600)) + .build(); + + return AssumeRoleResponse.builder() + .credentials(tmpSessionCredentials) + .build(); + }); + return mockStsClient; + }) + .when(credentialManager) + .getStsClient(any()); + } + + @Test + public void testGetConnectionCredentialsForSigV4() { + ServiceIdentityInfoDpo serviceIdentityInfo = + serviceIdentityRegistry.assignServiceIdentity(ServiceIdentityType.AWS_IAM); + EnumMap credentials = + credentialManager.getConnectionCredentials( + serviceIdentityInfo, + new SigV4AuthenticationParametersDpo( + "arn:aws:iam::123456789012:role/polaris-users-iam-role", + null, + null, + "us-west-2", + "glue")); + Assertions.assertThat(credentials) + .containsEntry(ConnectionCredentialProperty.AWS_ACCESS_KEY_ID, "tmp-access-key-id") + .containsEntry(ConnectionCredentialProperty.AWS_SECRET_ACCESS_KEY, "tmp-secret-access-key") + .containsEntry(ConnectionCredentialProperty.AWS_SESSION_TOKEN, "tmp-session-token") + .containsKey(ConnectionCredentialProperty.EXPIRATION_TIME) + .size() + .isEqualTo(4); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationConfiguration.java index 9301cadcce..e3a5a22272 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationConfiguration.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationConfiguration.java @@ -23,5 +23,5 @@ import java.util.Optional; public interface EntityMutationConfiguration { - Optional> mutators(); + Optional> mutators(); } From d2636dafc4e9d32497bf9993897afad1b566f281 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Wed, 4 Jun 2025 11:36:13 -0700 Subject: [PATCH 13/15] Add more comments and reorganize the structure --- .../connection/ConnectionConfigInfoDpo.java | 8 +++-- .../SigV4AuthenticationParametersDpo.java | 6 ++++ .../hadoop/HadoopConnectionConfigInfoDpo.java | 1 + .../IcebergRestConnectionConfigInfoDpo.java | 3 +- .../DefaultPolarisCredentialManager.java | 14 +++++++- .../credentials/PolarisCredentialManager.java | 22 ++++++++++++ .../ConnectionCredentialProperty.java | 6 ++++ .../PolarisConnectionCredsVendor.java | 28 +++++++++++++++ .../dpo/AwsIamServiceIdentityInfoDpo.java | 11 ++++++ .../identity/dpo/ServiceIdentityInfoDpo.java | 4 +++ .../mutation/EntityMutationEngine.java | 15 ++++++++ .../identity/mutation/EntityMutator.java | 15 +++++++- .../DefaultServiceIdentityRegistry.java | 18 ++++++++++ .../registry/ServiceIdentityRegistry.java | 26 ++++++++++++-- .../ServiceIdentityRegistryFactory.java | 6 ++++ .../ResolvedAwsIamServiceIdentity.java | 20 +++++++++++ .../resolved/ResolvedServiceIdentity.java | 1 + .../mutation/NoOpEntityMutationEngine.java | 5 +++ .../src/main/resources/application.properties | 3 +- ...PolarisCredentialManagerConfiguration.java | 7 ++++ .../CatalogEntityConnectionConfigMutator.java | 22 ++++++++++-- .../identity/mutation/NoOpEntityMutator.java | 7 +++- .../QuarkusEntityMutationConfiguration.java | 10 ++++++ .../AwsIamServiceIdentityConfiguration.java | 27 ++++++++++++++ .../RealmServiceIdentityConfiguration.java | 18 ++++++++++ ...esolvableServiceIdentityConfiguration.java | 15 ++++++++ .../ServiceIdentityConfiguration.java | 35 +++++++++++++++++++ .../mutation/EntityMutationEngineImpl.java | 1 + ...DefaultServiceIdentityRegistryFactory.java | 5 ++- .../apache/polaris/service/TestServices.java | 2 +- 30 files changed, 347 insertions(+), 14 deletions(-) rename {service/common/src/main/java/org/apache/polaris/service => polaris-core/src/main/java/org/apache/polaris/core}/identity/mutation/EntityMutator.java (71%) rename service/common/src/main/java/org/apache/polaris/service/identity/{ => registry}/DefaultServiceIdentityRegistryFactory.java (95%) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java index 97ffa9ed05..a6e95def54 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java @@ -18,7 +18,11 @@ */ package org.apache.polaris.core.connection; -import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -195,7 +199,7 @@ public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( } public abstract ConnectionConfigInfoDpo withServiceIdentity( - ServiceIdentityInfoDpo serviceIdentityInfo); + @Nonnull ServiceIdentityInfoDpo serviceIdentityInfo); /** * Produces the correponding API-model ConnectionConfigInfo for this persistence object; many diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java index e10f1f213f..ed48ca356a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/SigV4AuthenticationParametersDpo.java @@ -39,18 +39,24 @@ */ public class SigV4AuthenticationParametersDpo extends AuthenticationParametersDpo { + // The aws IAM role arn assumed by polaris userArn when signing requests @JsonProperty(value = "roleArn") private final String roleArn; + // The session name used when assuming the role @JsonProperty(value = "roleSessionName") private final String roleSessionName; + // An optional external id used to establish a trust relationship with AWS in the trust policy @JsonProperty(value = "externalId") private final String externalId; + // Region to be used by the SigV4 protocol for signing requests @JsonProperty(value = "signingRegion") private final String signingRegion; + // The service name to be used by the SigV4 protocol for signing requests, the default signing + // name is "execute-api" is if not provided @JsonProperty(value = "signingName") private final String signingName; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java index 90afbf6bf4..4d4432eeb3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/hadoop/HadoopConnectionConfigInfoDpo.java @@ -65,6 +65,7 @@ public String toString() { .add("uri", getUri()) .add("warehouse", getWarehouse()) .add("authenticationParameters", getAuthenticationParameters().toString()) + .add("serviceIdentity", getServiceIdentity()) .toString(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java index eb3bcb791a..a8fdf2c73f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/iceberg/IcebergRestConnectionConfigInfoDpo.java @@ -80,7 +80,8 @@ public String getRemoteCatalogName() { } @Override - public ConnectionConfigInfoDpo withServiceIdentity(ServiceIdentityInfoDpo serviceIdentityInfo) { + public ConnectionConfigInfoDpo withServiceIdentity( + @Nonnull ServiceIdentityInfoDpo serviceIdentityInfo) { return new IcebergRestConnectionConfigInfoDpo( getUri(), getAuthenticationParameters(), serviceIdentityInfo, getRemoteCatalogName()); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java index ef4e203625..a3b259453b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/DefaultPolarisCredentialManager.java @@ -20,6 +20,7 @@ package org.apache.polaris.core.credentials; import com.google.common.annotations.VisibleForTesting; +import jakarta.annotation.Nonnull; import java.util.EnumMap; import java.util.Optional; import org.apache.polaris.core.connection.AuthenticationParametersDpo; @@ -35,6 +36,17 @@ import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; +/** + * Default implementation of {@link PolarisCredentialManager} responsible for retrieving credentials + * used by Polaris to access external systems such as remote catalogs or cloud storage. + * + *

It resolves a {@link ServiceIdentityInfoDpo} into a {@link ResolvedServiceIdentity} using the + * {@link ServiceIdentityRegistry}, then uses the provided authentication parameters to generate + * temporary access credentials (e.g., via AWS STS AssumeRole). + * + *

This implementation currently supports AWS IAM service identities and can be extended to + * support other identity types or external services beyond catalogs, such as cloud storage. + */ public class DefaultPolarisCredentialManager implements PolarisCredentialManager { private final ServiceIdentityRegistry serviceIdentityRegistry; @@ -43,7 +55,7 @@ public DefaultPolarisCredentialManager(ServiceIdentityRegistry serviceIdentityRe } @Override - public EnumMap getConnectionCredentials( + public @Nonnull EnumMap getConnectionCredentials( ServiceIdentityInfoDpo serviceIdentity, AuthenticationParametersDpo authenticationParameters) { EnumMap credentialMap = diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManager.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManager.java index 568b1b7675..1039b3b3eb 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/PolarisCredentialManager.java @@ -19,14 +19,36 @@ package org.apache.polaris.core.credentials; +import jakarta.annotation.Nonnull; import java.util.EnumMap; import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.credentials.connection.ConnectionCredentialProperty; import org.apache.polaris.core.credentials.connection.PolarisConnectionCredsVendor; import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +/** + * PolarisCredentialManager is responsible for retrieving the credentials Polaris needs to access + * remote services such as federated catalogs or cloud storage. + * + *

It combines service-managed identity information (e.g., an IAM user Polaris uses) with + * user-defined authentication parameters (e.g., roleArn) to generate the credentials required for + * authentication with external systems. + * + *

Typical flow: + * + *

    + *
  1. Resolve the service identity and locate its associated credential (e.g., from a secret + * manager via the service identity registry). + *
  2. Use the resolved identity together with the authentication parameters to obtain the final + * access credentials. + *
+ * + *

This design supports both SaaS and self-managed deployments, ensuring a clear separation + * between user-provided configuration and Polaris-managed identity. + */ public interface PolarisCredentialManager extends PolarisConnectionCredsVendor { @Override + @Nonnull EnumMap getConnectionCredentials( ServiceIdentityInfoDpo serviceIdentity, AuthenticationParametersDpo authenticationParameters); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/ConnectionCredentialProperty.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/ConnectionCredentialProperty.java index fe99a9e739..b2d9133318 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/ConnectionCredentialProperty.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/ConnectionCredentialProperty.java @@ -21,6 +21,12 @@ import org.apache.iceberg.aws.AwsProperties; +/** + * A subset of Iceberg catalog properties recognized by Polaris. + * + *

Most of these properties are meant to initialize Catalog objects for accessing the remote + * Catalog service. + */ public enum ConnectionCredentialProperty { AWS_ACCESS_KEY_ID(String.class, AwsProperties.REST_ACCESS_KEY_ID, "the aws access key id"), AWS_SECRET_ACCESS_KEY( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/PolarisConnectionCredsVendor.java b/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/PolarisConnectionCredsVendor.java index 5ee892a7cf..8482f49f5f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/PolarisConnectionCredsVendor.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/credentials/connection/PolarisConnectionCredsVendor.java @@ -19,11 +19,39 @@ package org.apache.polaris.core.credentials.connection; +import jakarta.annotation.Nonnull; import java.util.EnumMap; import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +/** + * Generates credentials Polaris uses to connect to external catalog services such as AWS Glue or + * other federated endpoints. Implementations combine service-managed identity metadata (such as an + * IAM user or role, defined in {@link ServiceIdentityInfoDpo}) with user-provided authentication + * parameters (such as a role ARN or external ID, defined in {@link AuthenticationParametersDpo}) to + * construct a credential map consumable by Polaris. + * + *

This interface allows pluggable behavior for different authentication mechanisms and service + * vendors. Implementations can support service-specific credential provisioning or caching + * strategies as needed. + */ public interface PolarisConnectionCredsVendor { + + /** + * Retrieve credential values required to authenticate a remote connection. + * + *

The returned credentials are derived using the combination of Polaris-managed service + * identity and user-specified connection authentication parameters. Implementations may look up + * the service identity credential from a secret store or local config, and use it in conjunction + * with user-supplied data to produce scoped credentials for accessing remote services. + * + * @param serviceIdentity Polaris-managed identity metadata, including a reference to the backing + * credential (e.g., a secret ARN or ID) + * @param authenticationParameters Authentication configuration supplied by the Polaris user + * @return A map from {@link ConnectionCredentialProperty} to the resolved credential value, used + * by downstream systems to establish the connection + */ + @Nonnull EnumMap getConnectionCredentials( ServiceIdentityInfoDpo serviceIdentity, AuthenticationParametersDpo authenticationParameters); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/AwsIamServiceIdentityInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/AwsIamServiceIdentityInfoDpo.java index 0f0f56d7ae..2fa6cae244 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/AwsIamServiceIdentityInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/AwsIamServiceIdentityInfoDpo.java @@ -30,6 +30,17 @@ import org.apache.polaris.core.identity.ServiceIdentityType; import org.apache.polaris.core.secrets.ServiceSecretReference; +/** + * Persistence-layer representation of an AWS IAM service identity used by Polaris. + * + *

This class models an AWS IAM identity (either a user or role) and extends {@link + * ServiceIdentityInfoDpo}. It is typically used internally to store both the identity metadata + * (such as the IAM ARN) and a reference to the actual credential (e.g., via {@link + * ServiceSecretReference}). + * + *

Instances of this class are convertible to the public API model {@link + * AwsIamServiceIdentityInfo}. + */ public class AwsIamServiceIdentityInfoDpo extends ServiceIdentityInfoDpo { // Technically, it should be ^arn:(aws|aws-cn|aws-us-gov):iam::(\d{12}):(user|role)/.+$, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/ServiceIdentityInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/ServiceIdentityInfoDpo.java index 9eacf10a4e..ff8da6166d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/ServiceIdentityInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/dpo/ServiceIdentityInfoDpo.java @@ -68,6 +68,10 @@ public ServiceSecretReference getIdentityInfoReference() { return identityInfoReference; } + /** + * Converts this persistence object to the corresponding API model. During the conversion, some + * fields will be dropped, e.g. the reference to the service identity's credential + */ public abstract @Nonnull ServiceIdentityInfo asServiceIdentityInfoModel(); @Override diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java index 9094a09fc3..7027a6b8b2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java @@ -21,6 +21,21 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; +/** + * Engine responsible for applying a sequence of {@link EntityMutator} transformations to a {@link + * PolarisBaseEntity}. + * + *

This abstraction allows Polaris to customize or enrich entities during runtime or persistence, + * based on configured or contextual logic (e.g., injecting service identity info, computing derived + * fields). + */ public interface EntityMutationEngine { + /** + * Applies all registered entity mutators to the provided entity, in order. + * + * @param entity The original Polaris entity to mutate. + * @return A new or modified instance of {@link PolarisBaseEntity} after all mutations are + * applied. + */ PolarisBaseEntity applyMutations(PolarisBaseEntity entity); } diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutator.java similarity index 71% rename from service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java rename to polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutator.java index 261515999e..6b7b875b37 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutator.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutator.java @@ -17,10 +17,23 @@ * under the License. */ -package org.apache.polaris.service.identity.mutation; +package org.apache.polaris.core.identity.mutation; import org.apache.polaris.core.entity.PolarisBaseEntity; +/** + * A transformation hook that mutates a Polaris entity. + * + *

Implementations of this interface apply custom logic to modify or enrich a {@link + * PolarisBaseEntity}. + */ public interface EntityMutator { + + /** + * Applies the mutation logic to the given entity. + * + * @param entity the entity to mutate + * @return the mutated entity + */ PolarisBaseEntity apply(PolarisBaseEntity entity); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java index a30e2d55b0..50f9d3aef0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/DefaultServiceIdentityRegistry.java @@ -27,9 +27,27 @@ import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; +/** + * Default implementation of {@link ServiceIdentityRegistry} that resolves service identities from + * statically configured values (typically defined via Quarkus server configuration). + * + *

This implementation supports both multi-tenant (e.g., SaaS) and self-managed (single-tenant) + * Polaris deployments: + * + *

    + *
  • In multi-tenant mode, each tenant (realm) can have its own set of service identities + * defined in the configuration. The same identity will consistently be assigned for each + * {@link ServiceIdentityType} within a given tenant. + *
  • In single-tenant or self-managed deployments, a single set of service identities can be + * defined and used system-wide. + *
+ */ public class DefaultServiceIdentityRegistry implements ServiceIdentityRegistry { + /** Map of service identity types to their resolved identities. */ private final EnumMap resolvedServiceIdentities; + + /** Map of identity info references (URNs) to their resolved service identities. */ private final Map referenceToResolvedServiceIdentity; public DefaultServiceIdentityRegistry( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistry.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistry.java index e81fcb2007..05f512715d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistry.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistry.java @@ -23,14 +23,34 @@ import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; +/** + * A registry interface for managing and resolving service identities in Polaris. + * + *

In a multi-tenant Polaris deployment, each catalog or tenant may be associated with a distinct + * service identity that represents the Polaris service itself when accessing external systems + * (e.g., cloud services like AWS or GCP). This registry provides a central mechanism to manage + * those identities and resolve them at runtime. + * + *

The registry helps abstract the configuration and retrieval of service-managed credentials + * from the logic that uses them. It ensures a consistent and secure way to handle identity + * resolution across different deployment models, including SaaS and self-managed environments. + */ public interface ServiceIdentityRegistry { + /** + * Assigns a new {@link ServiceIdentityInfoDpo} for the given service identity type. Typically + * used during entity creation to associate a default or generated identity. + * + * @param serviceIdentityType The type of service identity (e.g., AWS_IAM). + * @return A new {@link ServiceIdentityInfoDpo} representing the assigned service identity. + */ ServiceIdentityInfoDpo assignServiceIdentity(ServiceIdentityType serviceIdentityType); /** - * Resolves the service identity based on the provided service identity information. + * Resolves the given service identity by retrieving the actual credential or secret referenced by + * it, typically from a secret manager or internal credential store. * - * @param serviceIdentityInfo The service identity information to resolve. - * @return The resolved service identity. + * @param serviceIdentityInfo The service identity metadata to resolve. + * @return A {@link ResolvedServiceIdentity} including credentials and other resolved data. */ ResolvedServiceIdentity resolveServiceIdentity(ServiceIdentityInfoDpo serviceIdentityInfo); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistryFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistryFactory.java index 385f6b5d77..b6036d5207 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistryFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/registry/ServiceIdentityRegistryFactory.java @@ -21,6 +21,12 @@ import org.apache.polaris.core.context.RealmContext; +/** + * Factory for creating {@link ServiceIdentityRegistry} instances. + * + *

Each {@link ServiceIdentityRegistry} instance is associated with a {@link RealmContext} and is + * responsible for managing the service identities for the user in that realm. + */ public interface ServiceIdentityRegistryFactory { ServiceIdentityRegistry getOrCreateServiceIdentityRegistry(RealmContext realmContext); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java index ffec4a89e1..9becd4c35d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedAwsIamServiceIdentity.java @@ -31,10 +31,29 @@ import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.StsClientBuilder; +/** + * Represents a fully resolved AWS IAM service identity, including the associated IAM ARN and + * credentials. This class is used internally by Polaris to access AWS services on behalf of a + * configured service identity. + * + *

It contains AWS credentials (access key, secret, and optional session token) and provides a + * lazily initialized {@link StsClient} for performing role assumptions or identity verification. + * + *

The resolved identity can be converted back into its persisted DPO form using {@link + * #asServiceIdentityInfoDpo()}. + */ public class ResolvedAwsIamServiceIdentity extends ResolvedServiceIdentity { + + /** IAM role or user ARN representing the Polaris service identity. */ private final String iamArn; + + /** AWS access key ID of the AWS credential associated with the identity. */ private final String accessKeyId; + + /** AWS secret access key of the AWS credential associated with the identity. */ private final String secretAccessKey; + + /** The AWS session token of the AWS credential associated with the identity. */ private final String sessionToken; public ResolvedAwsIamServiceIdentity( @@ -77,6 +96,7 @@ public ServiceIdentityInfoDpo asServiceIdentityInfoDpo() { return new AwsIamServiceIdentityInfoDpo(getIdentityInfoReference(), getIamArn()); } + /** Returns a memoized supplier for creating an STS client using the resolved credentials. */ public Supplier stsClientSupplier() { return Suppliers.memoize( () -> { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedServiceIdentity.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedServiceIdentity.java index ad981570c1..74b6a9516c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedServiceIdentity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/resolved/ResolvedServiceIdentity.java @@ -57,5 +57,6 @@ public void setIdentityInfoReference(@NotNull ServiceSecretReference identityInf this.identityInfoReference = identityInfoReference; } + /** Converts this resolved identity into its corresponding persisted form (DPO). */ public abstract @Nonnull ServiceIdentityInfoDpo asServiceIdentityInfoDpo(); } diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java index 58f0f4525f..e73f863ad6 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java @@ -21,6 +21,11 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; +/** + * A no-op implementation of {@link EntityMutationEngine} that returns the input entity unchanged. + * + *

This can be used in environments where entity mutation is disabled or unnecessary. + */ public class NoOpEntityMutationEngine implements EntityMutationEngine { @Override diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index 7a9e51d93d..48814c43e6 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -110,7 +110,7 @@ polaris.realm-context.require-header=false polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"] -polaris.features."ENABLE_CATALOG_FEDERATION"=true +# polaris.features."ENABLE_CATALOG_FEDERATION"=true polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] # realm overrides @@ -188,6 +188,7 @@ polaris.oidc.principal-roles-mapper.type=default # polaris.storage.gcp.token=token # polaris.storage.gcp.lifespan=PT1H +# Polaris Entity Mutation Config polaris.entity-mutation.mutators=no-op,catalog-connection-config # Polaris Service Identity Config diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credentials/QuarkusPolarisCredentialManagerConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credentials/QuarkusPolarisCredentialManagerConfiguration.java index 77e58c50ee..4f313581cc 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credentials/QuarkusPolarisCredentialManagerConfiguration.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/credentials/QuarkusPolarisCredentialManagerConfiguration.java @@ -22,6 +22,13 @@ import io.quarkus.runtime.annotations.StaticInitSafe; import io.smallrye.config.ConfigMapping; +/** + * Quarkus configuration mapping for Polaris Credential Manager. + * + *

Defines which {@link org.apache.polaris.core.credentials.PolarisCredentialManagerFactory} + * implementation should be used at runtime. This allows switching between different credential + * management strategies via configuration. + */ @StaticInitSafe @ConfigMapping(prefix = "polaris.credential-manager") public interface QuarkusPolarisCredentialManagerConfiguration { diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java index b35add0a80..700b5056f5 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java @@ -28,20 +28,36 @@ import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.identity.ServiceIdentityType; +import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +import org.apache.polaris.core.identity.mutation.EntityMutator; import org.apache.polaris.core.identity.registry.ServiceIdentityRegistry; -import org.apache.polaris.service.identity.mutation.EntityMutator; +/** + * Entity mutator that injects a {@link ServiceIdentityInfoDpo} into a passthrough {@link + * CatalogEntity}, allowing the Polaris service identity to be surfaced to Polaris users. + * + *

This is necessary for authentication mechanisms such as SigV4, where Polaris users must + * explicitly allowlist the service identity that Polaris uses to access their services. + */ @RequestScoped @Identifier("catalog-connection-config") public class CatalogEntityConnectionConfigMutator implements EntityMutator { - ServiceIdentityRegistry serviceIdentityRegistry; + private final ServiceIdentityRegistry serviceIdentityRegistry; @Inject CatalogEntityConnectionConfigMutator(ServiceIdentityRegistry serviceIdentityRegistry) { this.serviceIdentityRegistry = serviceIdentityRegistry; } + /** + * If the entity is a passthrough {@link CatalogEntity} and its connection uses AWS SIGV4 + * authentication, this method injects a service identity into its connection configuration. + * + * @param entity the original Polaris entity + * @return the mutated entity with service identity injected, or the original if no change is + * needed + */ @Override public PolarisBaseEntity apply(PolarisBaseEntity entity) { if (!(entity instanceof CatalogEntity catalogEntity) || !catalogEntity.isPassthroughFacade()) { @@ -51,6 +67,7 @@ public PolarisBaseEntity apply(PolarisBaseEntity entity) { ConnectionConfigInfoDpo connectionConfigInfoDpo = catalogEntity.getConnectionConfigInfoDpo(); AuthenticationParametersDpo authenticationParameters = connectionConfigInfoDpo.getAuthenticationParameters(); + if (authenticationParameters.getAuthenticationType() == AuthenticationType.SIGV4) { CatalogEntity.Builder builder = new CatalogEntity.Builder(catalogEntity); ConnectionConfigInfoDpo injectedConnectionConfigInfoDpo = @@ -60,6 +77,7 @@ public PolarisBaseEntity apply(PolarisBaseEntity entity) { builder.setEntityVersion(entity.getEntityVersion() + 1); return builder.build(); } + return entity; } } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java index b317b62391..b71db64323 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java @@ -22,8 +22,13 @@ import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.ApplicationScoped; import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.service.identity.mutation.EntityMutator; +import org.apache.polaris.core.identity.mutation.EntityMutator; +/** + * A no-op implementation of {@link EntityMutator} that returns the entity unchanged. + * + *

This can be used as a placeholder or for testing purposes when no mutation is required. + */ @ApplicationScoped @Identifier("no-op") public class NoOpEntityMutator implements EntityMutator { diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/QuarkusEntityMutationConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/QuarkusEntityMutationConfiguration.java index 04f58b4dce..d656dcc367 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/QuarkusEntityMutationConfiguration.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/QuarkusEntityMutationConfiguration.java @@ -22,8 +22,18 @@ import io.smallrye.config.ConfigMapping; import java.util.List; import java.util.Optional; +import org.apache.polaris.core.identity.mutation.EntityMutator; import org.apache.polaris.service.identity.mutation.EntityMutationConfiguration; +/** + * Quarkus-specific configuration interface for entity mutation behavior. + * + *

This configuration determines which {@link EntityMutator}s are applied during entity mutation + * and in what order. Only the listed mutators will be used, and they will be executed sequentially + * as configured. + * + *

If no mutators are specified, entity mutation is effectively disabled. + */ @ConfigMapping(prefix = "polaris.entity-mutation") public interface QuarkusEntityMutationConfiguration extends EntityMutationConfiguration { @Override diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/AwsIamServiceIdentityConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/AwsIamServiceIdentityConfiguration.java index cba3bf9b20..d69459f492 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/AwsIamServiceIdentityConfiguration.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/AwsIamServiceIdentityConfiguration.java @@ -22,15 +22,42 @@ import java.util.Optional; import org.apache.polaris.core.identity.resolved.ResolvedAwsIamServiceIdentity; +/** + * Configuration for an AWS IAM service identity used by Polaris to access AWS services. + * + *

This includes the IAM ARN and optionally, static credentials (access key, secret key, and + * session token). If credentials are provided, they will be used to construct a {@link + * ResolvedAwsIamServiceIdentity}; otherwise, the AWS default credential provider chain is used. + */ public interface AwsIamServiceIdentityConfiguration extends ResolvableServiceIdentityConfiguration { + + /** The IAM role or user ARN representing the service identity. */ String iamArn(); + /** + * Optional AWS access key ID associated with the IAM identity. If not provided, the AWS default + * credential chain will be used. + */ Optional accessKeyId(); + /** + * Optional AWS secret access key associated with the IAM identity. If not provided, the AWS + * default credential chain will be used. + */ Optional secretAccessKey(); + /** + * Optional AWS session token associated with the IAM identity. If not provided, the AWS default + * credential chain will be used. + */ Optional sessionToken(); + /** + * Resolves this configuration into a {@link ResolvedAwsIamServiceIdentity} if the IAM ARN is + * present. + * + * @return the resolved identity, or an empty optional if the ARN is missing + */ @Override default Optional resolve() { if (iamArn() == null) { diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/RealmServiceIdentityConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/RealmServiceIdentityConfiguration.java index 0c61d7ff6a..89d394f1b8 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/RealmServiceIdentityConfiguration.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/RealmServiceIdentityConfiguration.java @@ -23,9 +23,27 @@ import java.util.Optional; import java.util.stream.Stream; +/** + * Represents service identity configuration for a specific realm. + * + *

Supports multiple identity types, such as AWS IAM. This interface allows each realm to define + * the credentials and metadata needed to resolve service-managed identities. + */ public interface RealmServiceIdentityConfiguration { + + /** + * Returns the AWS IAM service identity configuration for this realm, if present. + * + * @return an optional AWS IAM configuration + */ Optional awsIamServiceIdentity(); + /** + * Aggregates all configured service identity types into a list. This includes AWS IAM and + * potentially other types in the future. + * + * @return a list of configured service identity definitions + */ default List serviceIdentityConfigurations() { return Stream.of(awsIamServiceIdentity()).flatMap(Optional::stream).toList(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/ResolvableServiceIdentityConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/ResolvableServiceIdentityConfiguration.java index 14b7651075..4b2bde9e54 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/ResolvableServiceIdentityConfiguration.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/ResolvableServiceIdentityConfiguration.java @@ -22,6 +22,21 @@ import java.util.Optional; import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; +/** + * Represents a service identity configuration that can be resolved into a fully initialized {@link + * ResolvedServiceIdentity}. + * + *

This interface allows identity configurations (e.g., AWS IAM) to encapsulate the logic + * required to construct runtime credentials and metadata needed to authenticate as a + * Polaris-managed service identity. + */ public interface ResolvableServiceIdentityConfiguration { + + /** + * Attempts to resolve this configuration into a {@link ResolvedServiceIdentity}. + * + * @return an optional resolved service identity, or empty if resolution fails or is not + * configured + */ Optional resolve(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/ServiceIdentityConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/identity/ServiceIdentityConfiguration.java index ec5e491476..a0a7e1a551 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/ServiceIdentityConfiguration.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/ServiceIdentityConfiguration.java @@ -22,15 +22,50 @@ import java.util.Map; import org.apache.polaris.core.context.RealmContext; +/** + * Represents the service identity configuration for one or more realms. + * + *

This interface supports multi-tenant configurations where each realm can define its own {@link + * RealmServiceIdentityConfiguration}. If a realm-specific configuration is not found, a fallback to + * the default configuration is applied. + * + * @param the type of per-realm service identity configuration + */ public interface ServiceIdentityConfiguration { + + /** + * The key used to identify the default realm configuration. + * + *

This default is especially useful in testing scenarios and single-tenant deployments where + * only one realm is expected and explicitly configuring realms is unnecessary. + */ String DEFAULT_REALM_KEY = ""; + /** + * Returns a map of realm identifiers to their corresponding service identity configurations. + * + * @return the map of realm-specific configurations + */ Map realms(); + /** + * Returns the service identity configuration for the given {@link RealmContext}. Falls back to + * the default if the realm is not explicitly configured. + * + * @param realmContext the realm context + * @return the matching or default realm configuration + */ default R forRealm(RealmContext realmContext) { return forRealm(realmContext.getRealmIdentifier()); } + /** + * Returns the service identity configuration for the given realm identifier. Falls back to the + * default if the realm is not explicitly configured. + * + * @param realmIdentifier the identifier of the realm + * @return the matching or default realm configuration + */ default R forRealm(String realmIdentifier) { return realms().containsKey(realmIdentifier) ? realms().get(realmIdentifier) diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java index 74c4396aad..23b4dccef0 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.mutation.EntityMutator; @ApplicationScoped public class EntityMutationEngineImpl implements EntityMutationEngine { diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java b/service/common/src/main/java/org/apache/polaris/service/identity/registry/DefaultServiceIdentityRegistryFactory.java similarity index 95% rename from service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java rename to service/common/src/main/java/org/apache/polaris/service/identity/registry/DefaultServiceIdentityRegistryFactory.java index f2df25ccc4..bde3b3da36 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/DefaultServiceIdentityRegistryFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/registry/DefaultServiceIdentityRegistryFactory.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.identity; +package org.apache.polaris.service.identity.registry; import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; @@ -33,6 +33,9 @@ import org.apache.polaris.core.identity.registry.ServiceIdentityRegistryFactory; import org.apache.polaris.core.identity.resolved.ResolvedServiceIdentity; import org.apache.polaris.core.secrets.ServiceSecretReference; +import org.apache.polaris.service.identity.RealmServiceIdentityConfiguration; +import org.apache.polaris.service.identity.ResolvableServiceIdentityConfiguration; +import org.apache.polaris.service.identity.ServiceIdentityConfiguration; @ApplicationScoped @Identifier("default") diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 31550c8edd..f5965851cd 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -67,7 +67,7 @@ import org.apache.polaris.service.credentials.DefaultPolarisCredentialManagerFactory; import org.apache.polaris.service.events.PolarisEventListener; import org.apache.polaris.service.events.TestPolarisEventListener; -import org.apache.polaris.service.identity.DefaultServiceIdentityRegistryFactory; +import org.apache.polaris.service.identity.registry.DefaultServiceIdentityRegistryFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.secrets.UnsafeInMemorySecretsManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; From b5bc2b59dcb7d9b166201df49ed904b14a691130 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Thu, 5 Jun 2025 21:45:58 -0700 Subject: [PATCH 14/15] Introduce MutationPoint-based filtering for EntityMutator --- .../mutation/EntityMutationEngine.java | 3 +- .../core/identity/mutation/MutationPoint.java | 43 +++++++++++++ .../TransactionalMetaStoreManagerImpl.java | 8 ++- .../mutation/NoOpEntityMutationEngine.java | 2 +- .../admintool/config/QuarkusProducers.java | 2 +- .../CatalogEntityConnectionConfigMutator.java | 3 + .../identity/mutation/NoOpEntityMutator.java | 1 + .../auth/JWTSymmetricKeyGeneratorTest.java | 3 +- .../service/identity/mutation/AppliesTo.java | 63 +++++++++++++++++++ .../service/identity/mutation/AppliesTos.java | 38 +++++++++++ .../mutation/EntityMutationEngineImpl.java | 18 +++--- 11 files changed, 171 insertions(+), 13 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/MutationPoint.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/mutation/AppliesTo.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/identity/mutation/AppliesTos.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java index 7027a6b8b2..a93b6c5f46 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java @@ -33,9 +33,10 @@ public interface EntityMutationEngine { /** * Applies all registered entity mutators to the provided entity, in order. * + * @param mutationPoint The point in the entity lifecycle where mutations should be applied. * @param entity The original Polaris entity to mutate. * @return A new or modified instance of {@link PolarisBaseEntity} after all mutations are * applied. */ - PolarisBaseEntity applyMutations(PolarisBaseEntity entity); + PolarisBaseEntity applyMutations(MutationPoint mutationPoint, PolarisBaseEntity entity); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/MutationPoint.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/MutationPoint.java new file mode 100644 index 0000000000..7300a89c35 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/MutationPoint.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.identity.mutation; + +/** + * Defines points in the entity lifecycle where {@link EntityMutator} can be applied. + * + *

Each mutation point corresponds to a specific hook where mutations may be executed. Mutators + * can declare which points they support, allowing the engine to invoke only the relevant ones. + */ +public enum MutationPoint { + + /** Applied before a catalog entity is persisted. */ + CATALOG_PRE_PERSIST(0), + ; + + private final int id; + + MutationPoint(int id) { + this.id = id; + } + + public int id() { + return id; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index 6e19e74879..a08437781f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -47,6 +47,8 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.mutation.MutationPoint; import org.apache.polaris.core.persistence.*; import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.ChangeTrackingResult; @@ -473,8 +475,10 @@ private void revokeGrantRecord( ms.persistStorageIntegrationIfNeededInCurrentTxn(callCtx, catalog, integration); - if (callCtx.getEntityMutationEngine() != null) { - catalog = callCtx.getEntityMutationEngine().applyMutations(catalog); + // CATALOG_PRE_PERSIST mutation point + EntityMutationEngine entityMutationEngine = callCtx.getEntityMutationEngine(); + if (entityMutationEngine != null) { + entityMutationEngine.applyMutations(MutationPoint.CATALOG_PRE_PERSIST, catalog); } // now create and persist new catalog entity diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java index e73f863ad6..159fb5a372 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/identity/mutation/NoOpEntityMutationEngine.java @@ -29,7 +29,7 @@ public class NoOpEntityMutationEngine implements EntityMutationEngine { @Override - public PolarisBaseEntity applyMutations(PolarisBaseEntity entity) { + public PolarisBaseEntity applyMutations(MutationPoint mutationPoint, PolarisBaseEntity entity) { return entity; } } diff --git a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java index 59d3619a13..9b8b641890 100644 --- a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java +++ b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java @@ -75,7 +75,7 @@ PolarisStorageIntegration getStorageIntegrationForConfig( @Produces public EntityMutationEngine entityMutationEngine() { // An entity mutation engine is not required when running the admin tool. - return entity -> entity; + return ((mutationPoint, entity) -> entity); } @Produces diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java index 700b5056f5..18448c4ef7 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java @@ -29,7 +29,9 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.identity.ServiceIdentityType; import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; +import org.apache.polaris.service.identity.mutation.AppliesTo; import org.apache.polaris.core.identity.mutation.EntityMutator; +import org.apache.polaris.core.identity.mutation.MutationPoint; import org.apache.polaris.core.identity.registry.ServiceIdentityRegistry; /** @@ -41,6 +43,7 @@ */ @RequestScoped @Identifier("catalog-connection-config") +@AppliesTo(MutationPoint.CATALOG_PRE_PERSIST) public class CatalogEntityConnectionConfigMutator implements EntityMutator { private final ServiceIdentityRegistry serviceIdentityRegistry; diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java index b71db64323..a2efac764c 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java @@ -23,6 +23,7 @@ import jakarta.enterprise.context.ApplicationScoped; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.identity.mutation.EntityMutator; +import org.apache.polaris.service.identity.mutation.AppliesTo; /** * A no-op implementation of {@link EntityMutator} that returns the entity unchanged. diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java index ff8a734584..9a6c40c8f5 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java @@ -45,7 +45,8 @@ public class JWTSymmetricKeyGeneratorTest { /** Sanity test to verify that we can generate a token */ @Test public void testJWTSymmetricKeyGenerator() { - PolarisCallContext polarisCallContext = new PolarisCallContext(null, null, null, null, null, null); + PolarisCallContext polarisCallContext = + new PolarisCallContext(null, null, null, null, null, null); PolarisMetaStoreManager metastoreManager = Mockito.mock(PolarisMetaStoreManager.class); String mainSecret = "test_secret"; String clientId = "test_client_id"; diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/AppliesTo.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/AppliesTo.java new file mode 100644 index 0000000000..f25dd03a0b --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/AppliesTo.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.identity.mutation; + +import jakarta.enterprise.util.AnnotationLiteral; +import jakarta.inject.Qualifier; +import java.lang.annotation.*; +import org.apache.polaris.core.identity.mutation.EntityMutationEngine; +import org.apache.polaris.core.identity.mutation.EntityMutator; +import org.apache.polaris.core.identity.mutation.MutationPoint; + +/** + * Qualifier to mark an {@link EntityMutator} as applicable to a specific {@link MutationPoint}. + * + *

This is used by the {@link EntityMutationEngine} to apply only relevant mutators based on + * context. + * + *

Supports being repeated on the same class to handle multiple mutation points. + */ +@Qualifier +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Repeatable(AppliesTos.class) +public @interface AppliesTo { + + /** The mutation point this mutator applies to. */ + MutationPoint value(); + + /** Helper for creating {@link AppliesTo} qualifiers programmatically. */ + final class Literal extends AnnotationLiteral implements AppliesTo { + private final MutationPoint value; + + public static Literal of(MutationPoint value) { + return new Literal(value); + } + + private Literal(MutationPoint value) { + this.value = value; + } + + @Override + public MutationPoint value() { + return value; + } + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/AppliesTos.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/AppliesTos.java new file mode 100644 index 0000000000..d3829ecb9f --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/AppliesTos.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.identity.mutation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.apache.polaris.core.identity.mutation.EntityMutator; +import org.apache.polaris.core.identity.mutation.MutationPoint; + +/** + * Container annotation for repeating {@link AppliesTo}. + * + *

Allows an {@link EntityMutator} to declare support for multiple {@link MutationPoint}s. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface AppliesTos { + AppliesTo[] value(); +} diff --git a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java index 23b4dccef0..849d3c2352 100644 --- a/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/identity/mutation/EntityMutationEngineImpl.java @@ -25,9 +25,11 @@ import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import java.util.List; +import java.util.Objects; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.identity.mutation.EntityMutationEngine; import org.apache.polaris.core.identity.mutation.EntityMutator; +import org.apache.polaris.core.identity.mutation.MutationPoint; @ApplicationScoped public class EntityMutationEngineImpl implements EntityMutationEngine { @@ -43,26 +45,28 @@ public EntityMutationEngineImpl( } @Override - public PolarisBaseEntity applyMutations(PolarisBaseEntity entity) { + public PolarisBaseEntity applyMutations(MutationPoint mutationPoint, PolarisBaseEntity entity) { PolarisBaseEntity result = entity; + // Collect mutators in configured order, filtering only those applicable to the mutation point List orderedMutators = - // config.map(EntityMutationConfiguration::mutators).orElse(List.of()).stream() config.mutators().orElse(List.of()).stream() .map( id -> { + // Resolve the mutator instance by ID Instance matched = mutatorInstances.select(Identifier.Literal.of(id)); - if (matched.isResolvable()) { - return matched.get(); - } else { + if (!matched.isResolvable()) { throw new IllegalStateException("No EntityMutator found for ID: " + id); } + // Filter by MutationPoint via @AppliesTo + Instance filtered = + matched.select(AppliesTo.Literal.of(mutationPoint)); + return filtered.isResolvable() ? filtered.get() : null; }) + .filter(Objects::nonNull) .toList(); - // Apply all mutators to the entity - // TODO: Add a way to control the order of mutators for (EntityMutator mutator : orderedMutators) { result = mutator.apply(result); } From d4479c7d0f9d1b03b8ef73017f280d98618b1e6f Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Mon, 9 Jun 2025 18:55:04 -0700 Subject: [PATCH 15/15] Reformat the code and fix some bugs --- .../polaris/core/identity/mutation/EntityMutationEngine.java | 4 ++-- .../apache/polaris/core/identity/mutation/EntityMutator.java | 3 ++- .../transactional/TransactionalMetaStoreManagerImpl.java | 2 +- .../mutation/CatalogEntityConnectionConfigMutator.java | 2 +- .../service/quarkus/identity/mutation/NoOpEntityMutator.java | 1 - 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java index a93b6c5f46..3ce1d06517 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutationEngine.java @@ -35,8 +35,8 @@ public interface EntityMutationEngine { * * @param mutationPoint The point in the entity lifecycle where mutations should be applied. * @param entity The original Polaris entity to mutate. - * @return A new or modified instance of {@link PolarisBaseEntity} after all mutations are - * applied. + * @return A new transformed copy of the entity of {@link PolarisBaseEntity} after all mutations + * are applied. */ PolarisBaseEntity applyMutations(MutationPoint mutationPoint, PolarisBaseEntity entity); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutator.java b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutator.java index 6b7b875b37..c29f443eb6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutator.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/identity/mutation/EntityMutator.java @@ -22,7 +22,8 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; /** - * A transformation hook that mutates a Polaris entity. + * A transformation hook that mutates a Polaris entity. The mutator must create a new transformed + * copy of the entity rather than mutating them in-place. * *

Implementations of this interface apply custom logic to modify or enrich a {@link * PolarisBaseEntity}. diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index a08437781f..71d7c3e7ed 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -478,7 +478,7 @@ private void revokeGrantRecord( // CATALOG_PRE_PERSIST mutation point EntityMutationEngine entityMutationEngine = callCtx.getEntityMutationEngine(); if (entityMutationEngine != null) { - entityMutationEngine.applyMutations(MutationPoint.CATALOG_PRE_PERSIST, catalog); + catalog = entityMutationEngine.applyMutations(MutationPoint.CATALOG_PRE_PERSIST, catalog); } // now create and persist new catalog entity diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java index 18448c4ef7..3d4fd2d688 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/CatalogEntityConnectionConfigMutator.java @@ -29,10 +29,10 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.identity.ServiceIdentityType; import org.apache.polaris.core.identity.dpo.ServiceIdentityInfoDpo; -import org.apache.polaris.service.identity.mutation.AppliesTo; import org.apache.polaris.core.identity.mutation.EntityMutator; import org.apache.polaris.core.identity.mutation.MutationPoint; import org.apache.polaris.core.identity.registry.ServiceIdentityRegistry; +import org.apache.polaris.service.identity.mutation.AppliesTo; /** * Entity mutator that injects a {@link ServiceIdentityInfoDpo} into a passthrough {@link diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java index a2efac764c..b71db64323 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/identity/mutation/NoOpEntityMutator.java @@ -23,7 +23,6 @@ import jakarta.enterprise.context.ApplicationScoped; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.identity.mutation.EntityMutator; -import org.apache.polaris.service.identity.mutation.AppliesTo; /** * A no-op implementation of {@link EntityMutator} that returns the entity unchanged.