From 16d33fd8b47fda3a78f34fbcb98909774981a29a Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Fri, 7 Feb 2025 21:20:55 +0000 Subject: [PATCH 01/17] Initial prototype of catalog federation just passing special properties into internal properties. Make Resolver federation-aware to properly handle "best-effort" resolution of passthrough facade entities. Targets will automatically reflect the longest-path that we happen to have stored locally and resolve grants against that path (including the degenerate case where the longest-path is just the catalog itself). This provides Catalog-level RBAC for passthrough federation. Sketch out persistence-layer flow for how connection secrets might be pushed down into a secrets-management layer. --- .../polaris/core/entity/CatalogEntity.java | 38 ++++++ .../polaris/core/entity/PolarisEntity.java | 5 + .../resolver/PolarisResolutionManifest.java | 26 +++- .../core/persistence/resolver/Resolver.java | 18 ++- .../TransactionalMetaStoreManagerImpl.java | 18 +++ .../iceberg/IcebergCatalogHandler.java | 114 ++++++++++++------ 6 files changed, 179 insertions(+), 40 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index a49bba9e6e..4d541e1042 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -66,6 +66,13 @@ public class CatalogEntity extends PolarisEntity { public static final String REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY = "replace-new-location-prefix-with-catalog-default"; + // TODO: Refactor all these into ConnectionConfigurationInfo + public static final String CONNECTION_REMOTE_URI_KEY = "connection.remoteUri"; + public static final String CONNECTION_CLIENT_ID_KEY = "connection.clientId"; + public static final String CONNECTION_CLIENT_SECRET_KEY = "connection.clientSecret"; + public static final String CONNECTION_CATALOG_NAME_KEY = "connection.catalogName"; + public static final String CONNECTION_SCOPES_KEY = "connection.scopes"; + public CatalogEntity(PolarisBaseEntity sourceEntity) { super(sourceEntity); } @@ -190,6 +197,37 @@ public Catalog.TypeEnum getCatalogType() { .orElse(null); } + public boolean isPassthroughFacade() { + // TODO: Refactor this to use new ConnectionConfigurationInfo + String remoteUri = getPropertiesAsMap().get(CONNECTION_REMOTE_URI_KEY); + return remoteUri != null && !remoteUri.isEmpty(); + } + + public String getConnectionRemoteUri() { + // TODO: Refactor this to use new ConnectionConfigurationInfo + return getPropertiesAsMap().get(CONNECTION_REMOTE_URI_KEY); + } + + public String getConnectionClientId() { + // TODO: Refactor this to use new ConnectionConfigurationInfo + return getInternalPropertiesAsMap().get(CONNECTION_CLIENT_ID_KEY); + } + + public String getConnectionClientSecret() { + // TODO: Refactor this to use new ConnectionConfigurationInfo + return getInternalPropertiesAsMap().get(CONNECTION_CLIENT_SECRET_KEY); + } + + public String getConnectionCatalogName() { + // TODO: Refactor this to use new ConnectionConfigurationInfo + return getPropertiesAsMap().get(CONNECTION_CATALOG_NAME_KEY); + } + + public String getConnectionScopes() { + // TODO: Refactor this to use new ConnectionConfigurationInfo + return getPropertiesAsMap().get(CONNECTION_SCOPES_KEY); + } + public static class Builder extends PolarisEntity.BaseBuilder { public Builder() { super(); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java index 08ac29b350..363c1d3ce4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java @@ -409,6 +409,11 @@ public B setInternalProperties(Map internalProperties) { return (B) this; } + public B addInternalProperty(String key, String value) { + this.internalProperties.put(key, value); + return (B) this; + } + public B setEntityVersion(int entityVersion) { this.entityVersion = entityVersion; return (B) this; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java index fd000a167a..4b2d5ce96e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java @@ -160,6 +160,10 @@ public ResolverStatus resolveAll() { return primaryResolverStatus; } + public boolean getIsPassthroughFacade() { + return primaryResolver.getIsPassthroughFacade(); + } + @Override public PolarisResolvedPathWrapper getResolvedReferenceCatalogEntity() { return getResolvedReferenceCatalogEntity(false); @@ -212,7 +216,12 @@ public PolarisResolvedPathWrapper getPassthroughResolvedPath(Object key) { } List resolvedPath = passthroughResolver.getResolvedPath(); - if (requestedPath.isOptional()) { + // If the catalog is a passthrough facade, we can go ahead and just return only as much of + // the parent path as was successfully found. + // TODO: For passthrough facade semantics, consider whether this should be where we generate + // the JIT-created entities that would get committed after we find them in the remote + // catalog. + if (requestedPath.isOptional() && !getIsPassthroughFacade()) { if (resolvedPath.size() != requestedPath.getEntityNames().size()) { LOGGER.debug( "Returning null for key {} due to size mismatch from getPassthroughResolvedPath " @@ -355,7 +364,12 @@ public PolarisResolvedPathWrapper getResolvedPath(Object key, boolean prependRoo // Return null for a partially-resolved "optional" path. ResolverPath requestedPath = addedPaths.get(index); List resolvedPath = primaryResolver.getResolvedPaths().get(index); - if (requestedPath.isOptional()) { + // If the catalog is a passthrough facade, we can go ahead and just return only as much of + // the parent path as was successfully found. + // TODO: For passthrough facade semantics, consider whether this should be where we generate + // the JIT-created entities that would get committed after we find them in the remote + // catalog. + if (requestedPath.isOptional() && !getIsPassthroughFacade()) { if (resolvedPath.size() != requestedPath.getEntityNames().size()) { return null; } @@ -383,7 +397,13 @@ public PolarisResolvedPathWrapper getResolvedPath( if (resolvedPath == null) { return null; } - if (resolvedPath.getRawLeafEntity() != null + // In the case of a passthrough facade, we may have only resolved part of the parent path + // in which case the subtype wouldn't match; return the path anyways in this case. + // + // TODO: Reconcile how we'll handle "TABLE_NOT_FOUND" or "VIEW_NOT_FOUND" semantics + // against the remote catalog. + if (!getIsPassthroughFacade() + && resolvedPath.getRawLeafEntity() != null && subType != PolarisEntitySubType.ANY_SUBTYPE && resolvedPath.getRawLeafEntity().getSubType() != subType) { return null; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java index b1c7fc1697..85d3e74c83 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java @@ -33,6 +33,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntityConstants; @@ -106,6 +107,10 @@ public class Resolver { private ResolverStatus resolverStatus; + // Set if we determine the reference catalog is a passthrough facade, which impacts + // leniency of resolution of in-catalog paths + private boolean isPassthroughFacade; + /** * Constructor, effectively starts an entity resolver session * @@ -264,6 +269,10 @@ public ResolverStatus resolveAll() { return status; } + public boolean getIsPassthroughFacade() { + return this.isPassthroughFacade; + } + /** * @return the principal we resolved */ @@ -656,6 +665,7 @@ private ResolverStatus resolveEntities( this.resolveByName(toValidate, entityName.getEntityType(), entityName.getEntityName()); // if not found, we can exit unless the entity is optional + // TODO: Consider how this interacts with CATALOG_ROLE in the isPassthroughFacade case. if (!entityName.isOptional() && (resolvedEntity == null || resolvedEntity.getEntity().isDropped())) { return new ResolverStatus(entityName.getEntityType(), entityName.getEntityName()); @@ -706,7 +716,9 @@ private ResolverStatus resolvePaths( // if not found, abort if (segment == null || segment.getEntity().isDropped()) { - if (path.isOptional()) { + // If we've determined the catalog is a passthrough facade, treat all paths as + // optional. + if (path.isOptional() || this.isPassthroughFacade) { // we have resolved as much as what we could have break; } else { @@ -851,6 +863,10 @@ private ResolverStatus resolveReferenceCatalog( } } + if (CatalogEntity.of(this.resolvedReferenceCatalog.getEntity()).isPassthroughFacade()) { + this.isPassthroughFacade = true; + } + // all good return new ResolverStatus(ResolverStatus.StatusEnum.SUCCESS); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index 27b049e917..b777a204b9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -33,6 +33,7 @@ import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; +import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -446,6 +447,23 @@ private void revokeGrantRecord( ms.persistStorageIntegrationIfNeededInCurrentTxn(callCtx, catalog, integration); + // TODO: Push this into an IntegrationPersistence flow which prepares an external secret store + // and swaps it with a reference in internalProperties which can be used to reconstitue the + // secret. + if (catalog.getPropertiesAsMap().containsKey(CatalogEntity.CONNECTION_CLIENT_SECRET_KEY)) { + catalog = + new PolarisEntity.Builder(PolarisEntity.of(catalog)) + .addInternalProperty( + CatalogEntity.CONNECTION_CLIENT_ID_KEY, + catalog.getPropertiesAsMap().get(CatalogEntity.CONNECTION_CLIENT_ID_KEY)) + .addInternalProperty( + CatalogEntity.CONNECTION_CLIENT_SECRET_KEY, + catalog.getPropertiesAsMap().get(CatalogEntity.CONNECTION_CLIENT_SECRET_KEY)) + .addProperty(CatalogEntity.CONNECTION_CLIENT_ID_KEY, "") + .addProperty(CatalogEntity.CONNECTION_CLIENT_SECRET_KEY, "") + .build(); + } + // now create and persist new catalog entity this.persistNewEntity(callCtx, ms, catalog); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index b5e1a0edb9..bcb377ebe7 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -19,6 +19,7 @@ package org.apache.polaris.service.catalog.iceberg; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import jakarta.ws.rs.core.SecurityContext; import java.io.Closeable; @@ -43,6 +44,7 @@ import org.apache.iceberg.UpdateRequirement; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SessionCatalog; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; @@ -52,6 +54,9 @@ import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.rest.CatalogHandlers; +import org.apache.iceberg.rest.HTTPClient; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; @@ -131,16 +136,6 @@ public IcebergCatalogHandler( this.catalogFactory = catalogFactory; } - @Override - protected void initializeCatalog() { - this.baseCatalog = - catalogFactory.createCallContextCatalog( - callContext, authenticatedPrincipal, securityContext, resolutionManifest); - this.namespaceCatalog = - (baseCatalog instanceof SupportsNamespaces) ? (SupportsNamespaces) baseCatalog : null; - this.viewCatalog = (baseCatalog instanceof ViewCatalog) ? (ViewCatalog) baseCatalog : null; - } - /** * TODO: Make the helper in org.apache.iceberg.rest.CatalogHandlers public instead of needing to * copy/paste here. @@ -162,6 +157,52 @@ public static boolean isCreate(UpdateTableRequest request) { return isCreate; } + @Override + protected void initializeCatalog() { + CatalogEntity resolvedCatalogEntity = + CatalogEntity.of(resolutionManifest.getResolvedReferenceCatalogEntity().getRawLeafEntity()); + if (resolvedCatalogEntity.getConnectionRemoteUri() != null) { + LOGGER + .atInfo() + .addKeyValue("remoteUrl", resolvedCatalogEntity.getConnectionRemoteUri()) + .log("Initializing federated catalog"); + + SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); + RESTCatalog restCatalog = + new RESTCatalog( + context, + (config) -> + HTTPClient.builder(config) + .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) + .build()); + + ImmutableMap.Builder propertiesBuilder = + ImmutableMap.builder() + .put( + org.apache.iceberg.CatalogProperties.URI, + resolvedCatalogEntity.getConnectionRemoteUri()) + .put( + OAuth2Properties.CREDENTIAL, + resolvedCatalogEntity.getConnectionClientId() + + ":" + + resolvedCatalogEntity.getConnectionClientSecret()) + .put(OAuth2Properties.SCOPE, resolvedCatalogEntity.getConnectionScopes()) + .put("warehouse", resolvedCatalogEntity.getConnectionCatalogName()); + + restCatalog.initialize( + resolvedCatalogEntity.getConnectionCatalogName(), propertiesBuilder.buildKeepingLast()); + this.baseCatalog = restCatalog; + } else { + LOGGER.atInfo().log("Initializing non-federated catalog"); + this.baseCatalog = + catalogFactory.createCallContextCatalog( + callContext, authenticatedPrincipal, securityContext, resolutionManifest); + } + this.namespaceCatalog = + (baseCatalog instanceof SupportsNamespaces) ? (SupportsNamespaces) baseCatalog : null; + this.viewCatalog = (baseCatalog instanceof ViewCatalog) ? (ViewCatalog) baseCatalog : null; + } + public ListNamespacesResponse listNamespaces(Namespace parent) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.LIST_NAMESPACES; authorizeBasicNamespaceOperationOrThrow(op, parent); @@ -203,9 +244,10 @@ public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { } } - private static boolean isExternal(CatalogEntity catalog) { + private static boolean isStaticFacade(CatalogEntity catalog) { return org.apache.polaris.core.admin.model.Catalog.TypeEnum.EXTERNAL.equals( - catalog.getCatalogType()); + catalog.getCatalogType()) + && !catalog.isPassthroughFacade(); } public GetNamespaceResponse loadNamespaceMetadata(Namespace namespace) { @@ -270,8 +312,8 @@ public LoadTableResponse createTableDirect(Namespace namespace, CreateTableReque .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create table on static-facade external catalogs."); } return CatalogHandlers.createTable(baseCatalog, namespace, request); } @@ -296,8 +338,8 @@ public LoadTableResponse createTableDirectWithWriteDelegation( .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create table on static-facade external catalogs."); } request.validate(); @@ -401,8 +443,8 @@ public LoadTableResponse createTableStaged(Namespace namespace, CreateTableReque .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create table on static-facade external catalogs."); } TableMetadata metadata = stageTableCreateHelper(namespace, request); return LoadTableResponse.builder().withTableMetadata(metadata).build(); @@ -421,8 +463,8 @@ public LoadTableResponse createTableStagedWithWriteDelegation( .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create table on static-facade external catalogs."); } TableIdentifier ident = TableIdentifier.of(namespace, request.name()); TableMetadata metadata = stageTableCreateHelper(namespace, request); @@ -696,8 +738,8 @@ public LoadTableResponse updateTable( .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot update table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot update table on static-facade external catalogs."); } return CatalogHandlers.updateTable(baseCatalog, tableIdentifier, applyUpdateFilters(request)); } @@ -713,8 +755,8 @@ public LoadTableResponse updateTableForStagedCreate( .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot update table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot update table on static-facade external catalogs."); } return CatalogHandlers.updateTable(baseCatalog, tableIdentifier, applyUpdateFilters(request)); } @@ -738,8 +780,8 @@ public void dropTableWithPurge(TableIdentifier tableIdentifier) { .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot drop table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot drop table on static-facade external catalogs."); } CatalogHandlers.purgeTable(baseCatalog, tableIdentifier); } @@ -764,8 +806,8 @@ public void renameTable(RenameTableRequest request) { .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot rename table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot rename table on static-facade external catalogs."); } CatalogHandlers.renameTable(baseCatalog, request); } @@ -788,8 +830,8 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot update table on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot update table on static-facade external catalogs."); } if (!(baseCatalog instanceof IcebergCatalog)) { @@ -894,8 +936,8 @@ public LoadViewResponse createView(Namespace namespace, CreateViewRequest reques .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot create view on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot create view on static-facade external catalogs."); } return CatalogHandlers.createView(viewCatalog, namespace, request); } @@ -917,8 +959,8 @@ public LoadViewResponse replaceView(TableIdentifier viewIdentifier, UpdateTableR .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot replace view on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot replace view on static-facade external catalogs."); } return CatalogHandlers.updateView(viewCatalog, viewIdentifier, applyUpdateFilters(request)); } @@ -949,8 +991,8 @@ public void renameView(RenameTableRequest request) { .getResolvedReferenceCatalogEntity() .getResolvedLeafEntity() .getEntity()); - if (isExternal(catalog)) { - throw new BadRequestException("Cannot rename view on external catalogs."); + if (isStaticFacade(catalog)) { + throw new BadRequestException("Cannot rename view on static-facade external catalogs."); } CatalogHandlers.renameView(viewCatalog, request); } From 3b1aa1681737512e524fdd66ce7d4813aaaec4d5 Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Mon, 3 Mar 2025 06:48:02 +0000 Subject: [PATCH 02/17] Defined internal representation classes for connection config --- .../core/connection/ConnectionType.java | 23 ++++ ...cebergRestConnectionConfigurationInfo.java | 63 ++++++++++ .../PolarisBearerRestAuthenticationInfo.java | 49 ++++++++ .../PolarisConnectionConfigurationInfo.java | 110 ++++++++++++++++++ .../PolarisOauthRestAuthenticationInfo.java | 109 +++++++++++++++++ .../PolarisRestAuthenticationInfo.java | 45 +++++++ .../connection/RestAuthenticationType.java | 24 ++++ ...olarisConnectionConfigurationInfoTest.java | 79 +++++++++++++ 8 files changed, 502 insertions(+) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/connection/RestAuthenticationType.java create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java new file mode 100644 index 0000000000..5be343d180 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +public enum ConnectionType { + ICEBERG_REST +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java new file mode 100644 index 0000000000..a0086f6122 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; + +public class IcebergRestConnectionConfigurationInfo extends PolarisConnectionConfigurationInfo { + + private final String remoteCatalogName; + + private final PolarisRestAuthenticationInfo restAuthentication; + + public IcebergRestConnectionConfigurationInfo( + @JsonProperty(value = "connectionType", required = true) @Nonnull + ConnectionType connectionType, + @JsonProperty(value = "remoteUri", required = true) @Nonnull String remoteUri, + @JsonProperty(value = "remoteCatalogName", required = false) @Nullable + String remoteCatalogName, + @JsonProperty(value = "restAuthentication", required = false) @Nonnull + PolarisRestAuthenticationInfo restAuthentication) { + super(connectionType, remoteUri); + this.remoteCatalogName = remoteCatalogName; + this.restAuthentication = restAuthentication; + } + + public String getRemoteCatalogName() { + return remoteCatalogName; + } + + public PolarisRestAuthenticationInfo getRestAuthentication() { + return restAuthentication; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("connectionType", getConnectionType()) + .add("connectionType", getConnectionType().name()) + .add("remoteUri", getRemoteUri()) + .add("remoteCatalogName", getRemoteCatalogName()) + .add("restAuthentication", getRestAuthentication().toString()) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java new file mode 100644 index 0000000000..5737927d81 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; + +public class PolarisBearerRestAuthenticationInfo extends PolarisRestAuthenticationInfo { + + @JsonProperty(value = "bearerToken") + private final String bearerToken; + + public PolarisBearerRestAuthenticationInfo( + @JsonProperty(value = "restAuthenticationType", required = true) @Nonnull + RestAuthenticationType restAuthenticationType, + @JsonProperty(value = "bearerToken", required = true) @Nonnull String bearerToken) { + super(restAuthenticationType); + this.bearerToken = bearerToken; + } + + public @Nonnull String getBearerToken() { + return bearerToken; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("authenticationType", getRestAuthenticationType()) + .add("bearerToken", getBearerToken()) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java new file mode 100644 index 0000000000..fabac6f946 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.Nonnull; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import org.apache.polaris.core.PolarisDiagnostics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "connectionType", visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = IcebergRestConnectionConfigurationInfo.class, name = "ICEBERG_REST"), +}) +public abstract class PolarisConnectionConfigurationInfo { + private static final Logger logger = + LoggerFactory.getLogger(PolarisConnectionConfigurationInfo.class); + + // The type of the connection + private final ConnectionType connectionType; + + // The URI of the remote catalog + private final String remoteUri; + + public PolarisConnectionConfigurationInfo( + @JsonProperty(value = "connectionType", required = true) @Nonnull + ConnectionType connectionType, + @JsonProperty(value = "remoteUri", required = true) @Nonnull String remoteUri) { + this(connectionType, remoteUri, true); + } + + protected PolarisConnectionConfigurationInfo( + ConnectionType connectionType, String remoteUri, boolean validateRemoteUri) { + this.connectionType = connectionType; + this.remoteUri = remoteUri; + if (validateRemoteUri) { + validateRemoteUri(remoteUri); + } + } + + public ConnectionType getConnectionType() { + return connectionType; + } + + public String getRemoteUri() { + return remoteUri; + } + + private static final ObjectMapper DEFAULT_MAPPER; + + static { + DEFAULT_MAPPER = new ObjectMapper(); + DEFAULT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); + DEFAULT_MAPPER.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + } + + public String serialize() { + try { + return DEFAULT_MAPPER.writeValueAsString(this); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static PolarisConnectionConfigurationInfo deserialize( + @Nonnull PolarisDiagnostics diagnostics, final @Nonnull String jsonStr) { + try { + return DEFAULT_MAPPER.readValue(jsonStr, PolarisConnectionConfigurationInfo.class); + } catch (JsonProcessingException exception) { + diagnostics.fail( + "fail_to_deserialize_connection_configuration", exception, "jsonStr={}", jsonStr); + } + return null; + } + + /** Validates the remote URI. */ + protected void validateRemoteUri(String remoteUri) { + try { + URI uri = URI.create(remoteUri); + URL url = uri.toURL(); + } catch (IllegalArgumentException | MalformedURLException e) { + throw new IllegalArgumentException("Invalid remote URI: " + remoteUri, e); + } + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java new file mode 100644 index 0000000000..a3af7f7701 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.util.List; +import java.util.Objects; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.iceberg.rest.auth.OAuth2Util; + +public class PolarisOauthRestAuthenticationInfo extends PolarisRestAuthenticationInfo { + @JsonProperty(value = "tokenUri") + private final String tokenUri; + + @JsonProperty(value = "clientId") + private final String clientId; + + @JsonProperty(value = "clientSecret") + private final String clientSecret; + + @JsonProperty(value = "scopes") + private final List scopes; + + public PolarisOauthRestAuthenticationInfo( + @JsonProperty(value = "restAuthenticationType", required = true) @Nonnull + RestAuthenticationType restAuthenticationType, + @JsonProperty(value = "tokenUri", required = false) @Nullable String tokenUri, + @JsonProperty(value = "clientId", required = true) @Nonnull String clientId, + @JsonProperty(value = "clientSecret", required = true) @Nonnull String clientSecret, + @JsonProperty(value = "scopes", required = false) @Nullable List scopes) { + super(restAuthenticationType); + + this.tokenUri = tokenUri; + this.clientId = clientId; + this.clientSecret = clientSecret; + this.scopes = scopes; + + validateTokenUri(tokenUri); + } + + public @Nonnull String getTokenUri() { + return tokenUri; + } + + public @Nonnull String getClientId() { + return clientId; + } + + public @Nonnull String getClientSecret() { + return clientSecret; + } + + public @Nonnull List getScopes() { + return scopes; + } + + @JsonIgnore + public @Nonnull String getScopesAsString() { + return OAuth2Util.toScope( + Objects.requireNonNullElse(scopes, List.of(OAuth2Properties.CATALOG_SCOPE))); + } + + /** Validates the token URI. */ + protected void validateTokenUri(String tokenUri) { + if (tokenUri == null) { + return; + } + + try { + URI uri = URI.create(tokenUri); + URL url = uri.toURL(); + } catch (IllegalArgumentException | MalformedURLException e) { + throw new IllegalArgumentException("Invalid token URI: " + tokenUri, e); + } + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("tokenUri", getTokenUri()) + .add("clientId", getClientId()) + .add("clientSecret", getClientSecret()) + .add("scopes", getScopesAsString()) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java new file mode 100644 index 0000000000..586ccc3e55 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.annotation.Nonnull; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "restAuthenticationType", visible = true) +@JsonSubTypes({ + @JsonSubTypes.Type(value = PolarisOauthRestAuthenticationInfo.class, name = "OAUTH"), + @JsonSubTypes.Type(value = PolarisBearerRestAuthenticationInfo.class, name = "BEARER"), +}) +public abstract class PolarisRestAuthenticationInfo { + + @JsonProperty(value = "restAuthenticationType") + private final RestAuthenticationType restAuthenticationType; + + public PolarisRestAuthenticationInfo( + @JsonProperty(value = "restAuthenticationType", required = true) @Nonnull + RestAuthenticationType restAuthenticationType) { + this.restAuthenticationType = restAuthenticationType; + } + + public @Nonnull RestAuthenticationType getRestAuthenticationType() { + return restAuthenticationType; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/RestAuthenticationType.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/RestAuthenticationType.java new file mode 100644 index 0000000000..dcbb623059 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/RestAuthenticationType.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +public enum RestAuthenticationType { + OAUTH, + BEARER +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java new file mode 100644 index 0000000000..38babf2d09 --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.polaris.core.PolarisDiagnostics; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class PolarisConnectionConfigurationInfoTest { + PolarisDiagnostics polarisDiagnostics = Mockito.mock(PolarisDiagnostics.class); + ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void testOauthRestAuthenticationInfo() throws JsonProcessingException { + String json = + "" + + "{" + + " \"connectionType\": \"ICEBERG_REST\"," + + " \"remoteUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"remoteCatalogName\": \"my-catalog\"," + + " \"restAuthentication\": {" + + " \"restAuthenticationType\": \"OAUTH\"," + + " \"tokenUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog/v1/oauth/tokens\"," + + " \"clientId\": \"client-id\"," + + " \"clientSecret\": \"client-secret\"," + + " \"scopes\": [\"PRINCIPAL_ROLE:ALL\"]" + + " }" + + "}"; + PolarisConnectionConfigurationInfo connectionConfigurationInfo = + PolarisConnectionConfigurationInfo.deserialize(polarisDiagnostics, json); + Assertions.assertNotNull(connectionConfigurationInfo); + System.out.println(connectionConfigurationInfo.serialize()); + JsonNode tree1 = objectMapper.readTree(json); + JsonNode tree2 = objectMapper.readTree(connectionConfigurationInfo.serialize()); + Assertions.assertEquals(tree1, tree2); + } + + @Test + void testBearerRestAuthenticationInfo() throws JsonProcessingException { + String json = + "" + + "{" + + " \"connectionType\": \"ICEBERG_REST\"," + + " \"remoteUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"remoteCatalogName\": \"my-catalog\"," + + " \"restAuthentication\": {" + + " \"restAuthenticationType\": \"BEARER\"," + + " \"bearerToken\": \"bearer-token\"" + + " }" + + "}"; + PolarisConnectionConfigurationInfo connectionConfigurationInfo = + PolarisConnectionConfigurationInfo.deserialize(polarisDiagnostics, json); + Assertions.assertNotNull(connectionConfigurationInfo); + System.out.println(connectionConfigurationInfo.serialize()); + JsonNode tree1 = objectMapper.readTree(json); + JsonNode tree2 = objectMapper.readTree(connectionConfigurationInfo.serialize()); + Assertions.assertEquals(tree1, tree2); + } +} From 1d1650b45d3e4a527f141fabc03c0cab92d9518e Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Mon, 3 Mar 2025 08:37:38 -0800 Subject: [PATCH 03/17] Construct and initialize federated iceberg catalog based on connection config --- .../IcebergCatalogPropertiesProvider.java | 27 ++++++++ ...cebergRestConnectionConfigurationInfo.java | 30 ++++++++- .../PolarisBearerRestAuthenticationInfo.java | 19 ++++++ .../PolarisConnectionConfigurationInfo.java | 6 +- .../PolarisOauthRestAuthenticationInfo.java | 39 +++++++++++- .../PolarisRestAuthenticationInfo.java | 42 ++++++++++++- .../polaris/core/entity/CatalogEntity.java | 61 ++++++++++++++++++- .../core/entity/PolarisEntityConstants.java | 7 +++ .../iceberg/IcebergCatalogHandler.java | 54 ++++++++-------- 9 files changed, 249 insertions(+), 36 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java new file mode 100644 index 0000000000..74953ae560 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.connection; + +import jakarta.annotation.Nonnull; +import java.util.Map; + +public interface IcebergCatalogPropertiesProvider { + @Nonnull + Map asIcebergCatalogProperties(); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java index a0086f6122..d1900722ed 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java @@ -22,8 +22,15 @@ import com.google.common.base.MoreObjects; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import org.apache.iceberg.CatalogProperties; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; +import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; +import org.jetbrains.annotations.NotNull; -public class IcebergRestConnectionConfigurationInfo extends PolarisConnectionConfigurationInfo { +public class IcebergRestConnectionConfigurationInfo extends PolarisConnectionConfigurationInfo + implements IcebergCatalogPropertiesProvider { private final String remoteCatalogName; @@ -50,6 +57,27 @@ public PolarisRestAuthenticationInfo getRestAuthentication() { return restAuthentication; } + @Override + public @NotNull Map asIcebergCatalogProperties() { + HashMap properties = new HashMap<>(); + properties.put(CatalogProperties.URI, getRemoteUri()); + if (getRemoteCatalogName() != null) { + properties.put(CatalogProperties.WAREHOUSE_LOCATION, getRemoteCatalogName()); + } + properties.putAll(restAuthentication.asIcebergCatalogProperties()); + return properties; + } + + @Override + public ConnectionConfigInfo asConnectionConfigInfoModel() { + return IcebergRestConnectionConfigInfo.builder() + .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) + .setRemoteUri(getRemoteUri()) + .setRemoteCatalogName(getRemoteCatalogName()) + .setRestAuthentication(restAuthentication.asRestAuthenticationInfoModel()) + .build(); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java index 5737927d81..acd339e7d6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java @@ -21,6 +21,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; import jakarta.annotation.Nonnull; +import java.util.Map; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.polaris.core.admin.model.BearerRestAuthenticationInfo; +import org.apache.polaris.core.admin.model.RestAuthenticationInfo; +import org.jetbrains.annotations.NotNull; public class PolarisBearerRestAuthenticationInfo extends PolarisRestAuthenticationInfo { @@ -39,6 +44,20 @@ public PolarisBearerRestAuthenticationInfo( return bearerToken; } + @Override + public @NotNull Map asIcebergCatalogProperties() { + return Map.of(OAuth2Properties.TOKEN, getBearerToken()); + } + + @Override + public RestAuthenticationInfo asRestAuthenticationInfoModel() { + // TODO: redact secrets from the model + return BearerRestAuthenticationInfo.builder() + .setRestAuthenticationType(RestAuthenticationInfo.RestAuthenticationTypeEnum.BEARER) + .setBearerToken(getBearerToken()) + .build(); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java index fabac6f946..83d57a5452 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java @@ -30,6 +30,7 @@ import java.net.URI; import java.net.URL; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +38,8 @@ @JsonSubTypes({ @JsonSubTypes.Type(value = IcebergRestConnectionConfigurationInfo.class, name = "ICEBERG_REST"), }) -public abstract class PolarisConnectionConfigurationInfo { +public abstract class PolarisConnectionConfigurationInfo + implements IcebergCatalogPropertiesProvider { private static final Logger logger = LoggerFactory.getLogger(PolarisConnectionConfigurationInfo.class); @@ -107,4 +109,6 @@ protected void validateRemoteUri(String remoteUri) { throw new IllegalArgumentException("Invalid remote URI: " + remoteUri, e); } } + + public abstract ConnectionConfigInfo asConnectionConfigInfoModel(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java index a3af7f7701..472bfc2349 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java @@ -20,18 +20,27 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.auth.OAuth2Util; +import org.apache.polaris.core.admin.model.OauthRestAuthenticationInfo; +import org.apache.polaris.core.admin.model.RestAuthenticationInfo; +import org.jetbrains.annotations.NotNull; public class PolarisOauthRestAuthenticationInfo extends PolarisRestAuthenticationInfo { + + private static final Joiner COLON_JOINER = Joiner.on(":"); + @JsonProperty(value = "tokenUri") private final String tokenUri; @@ -61,7 +70,7 @@ public PolarisOauthRestAuthenticationInfo( validateTokenUri(tokenUri); } - public @Nonnull String getTokenUri() { + public @Nullable String getTokenUri() { return tokenUri; } @@ -77,12 +86,40 @@ public PolarisOauthRestAuthenticationInfo( return scopes; } + @JsonIgnore + public @Nonnull String getCredential() { + return COLON_JOINER.join(clientId, clientSecret); + } + @JsonIgnore public @Nonnull String getScopesAsString() { return OAuth2Util.toScope( Objects.requireNonNullElse(scopes, List.of(OAuth2Properties.CATALOG_SCOPE))); } + @Override + public @NotNull Map asIcebergCatalogProperties() { + HashMap properties = new HashMap<>(); + if (getTokenUri() != null) { + properties.put(OAuth2Properties.OAUTH2_SERVER_URI, getTokenUri()); + } + properties.put(OAuth2Properties.CREDENTIAL, getCredential()); + properties.put(OAuth2Properties.SCOPE, getScopesAsString()); + return properties; + } + + @Override + public RestAuthenticationInfo asRestAuthenticationInfoModel() { + // TODO: redact secrets from the model + return OauthRestAuthenticationInfo.builder() + .setRestAuthenticationType(RestAuthenticationInfo.RestAuthenticationTypeEnum.OAUTH) + .setTokenUri(getTokenUri()) + .setClientId(getClientId()) + .setClientSecret(getClientSecret()) + .setScopes(getScopes()) + .build(); + } + /** Validates the token URI. */ protected void validateTokenUri(String tokenUri) { if (tokenUri == null) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java index 586ccc3e55..a1a48cad04 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java @@ -22,13 +22,18 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import jakarta.annotation.Nonnull; +import java.util.Map; +import org.apache.polaris.core.admin.model.BearerRestAuthenticationInfo; +import org.apache.polaris.core.admin.model.OauthRestAuthenticationInfo; +import org.apache.polaris.core.admin.model.RestAuthenticationInfo; +import org.jetbrains.annotations.NotNull; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "restAuthenticationType", visible = true) @JsonSubTypes({ @JsonSubTypes.Type(value = PolarisOauthRestAuthenticationInfo.class, name = "OAUTH"), @JsonSubTypes.Type(value = PolarisBearerRestAuthenticationInfo.class, name = "BEARER"), }) -public abstract class PolarisRestAuthenticationInfo { +public abstract class PolarisRestAuthenticationInfo implements IcebergCatalogPropertiesProvider { @JsonProperty(value = "restAuthenticationType") private final RestAuthenticationType restAuthenticationType; @@ -42,4 +47,39 @@ public PolarisRestAuthenticationInfo( public @Nonnull RestAuthenticationType getRestAuthenticationType() { return restAuthenticationType; } + + @Override + public abstract @NotNull Map asIcebergCatalogProperties(); + + public abstract RestAuthenticationInfo asRestAuthenticationInfoModel(); + + public static PolarisRestAuthenticationInfo fromRestAuthenticationInfoModel( + RestAuthenticationInfo restAuthenticationInfo) { + PolarisRestAuthenticationInfo config = null; + switch (restAuthenticationInfo.getRestAuthenticationType()) { + case OAUTH: + OauthRestAuthenticationInfo oauthRestAuthenticationModel = + (OauthRestAuthenticationInfo) restAuthenticationInfo; + config = + new PolarisOauthRestAuthenticationInfo( + RestAuthenticationType.OAUTH, + oauthRestAuthenticationModel.getTokenUri(), + oauthRestAuthenticationModel.getClientId(), + oauthRestAuthenticationModel.getClientSecret(), + oauthRestAuthenticationModel.getScopes()); + break; + case BEARER: + BearerRestAuthenticationInfo bearerRestAuthenticationModel = + (BearerRestAuthenticationInfo) restAuthenticationInfo; + config = + new PolarisBearerRestAuthenticationInfo( + RestAuthenticationType.BEARER, bearerRestAuthenticationModel.getBearerToken()); + break; + default: + throw new IllegalStateException( + "Unsupported authentication type: " + + restAuthenticationInfo.getRestAuthenticationType()); + } + return config; + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index 4d541e1042..31d41b060a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -41,6 +41,10 @@ import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.config.BehaviorChangeConfiguration; +import org.apache.polaris.core.connection.ConnectionType; +import org.apache.polaris.core.connection.IcebergRestConnectionConfigurationInfo; +import org.apache.polaris.core.connection.PolarisConnectionConfigurationInfo; +import org.apache.polaris.core.connection.PolarisRestAuthenticationInfo; import org.apache.polaris.core.storage.FileStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; @@ -96,6 +100,9 @@ public static CatalogEntity fromCatalog(Catalog catalog) { builder.setInternalProperties(internalProperties); builder.setStorageConfigurationInfo( catalog.getStorageConfigInfo(), getDefaultBaseLocation(catalog)); + if (catalog instanceof ExternalCatalog) { + builder.setConnectionConfigurationInfo(((ExternalCatalog) catalog).getConnectionConfigInfo()); + } return builder.build(); } @@ -128,6 +135,7 @@ public Catalog asCatalog() { .setLastUpdateTimestamp(getLastUpdateTimestamp()) .setEntityVersion(getEntityVersion()) .setStorageConfigInfo(getStorageInfo(internalProperties)) + .setConnectionConfigInfo(getConnectionInfo(internalProperties)) .build(); } @@ -173,6 +181,15 @@ private StorageConfigInfo getStorageInfo(Map internalProperties) return null; } + private ConnectionConfigInfo getConnectionInfo(Map internalProperties) { + if (internalProperties.containsKey( + PolarisEntityConstants.getConnectionConfigInfoPropertyName())) { + PolarisConnectionConfigurationInfo configInfo = getConnectionConfigurationInfo(); + return configInfo.asConnectionConfigInfoModel(); + } + return null; + } + public String getDefaultBaseLocation() { return getPropertiesAsMap().get(DEFAULT_BASE_LOCATION_KEY); } @@ -198,9 +215,19 @@ public Catalog.TypeEnum getCatalogType() { } public boolean isPassthroughFacade() { - // TODO: Refactor this to use new ConnectionConfigurationInfo - String remoteUri = getPropertiesAsMap().get(CONNECTION_REMOTE_URI_KEY); - return remoteUri != null && !remoteUri.isEmpty(); + return getInternalPropertiesAsMap() + .containsKey(PolarisEntityConstants.getConnectionConfigInfoPropertyName()); + } + + public PolarisConnectionConfigurationInfo getConnectionConfigurationInfo() { + String configStr = + getInternalPropertiesAsMap() + .get(PolarisEntityConstants.getConnectionConfigInfoPropertyName()); + if (configStr != null) { + return PolarisConnectionConfigurationInfo.deserialize( + new PolarisDefaultDiagServiceImpl(), configStr); + } + return null; } public String getConnectionRemoteUri() { @@ -328,6 +355,34 @@ private void validateMaxAllowedLocations(Collection allowedLocations) { } } + public Builder setConnectionConfigurationInfo( + ConnectionConfigInfo connectionConfigurationModel) { + if (connectionConfigurationModel != null) { + PolarisConnectionConfigurationInfo config; + switch (connectionConfigurationModel.getConnectionType()) { + case ICEBERG_REST: + IcebergRestConnectionConfigInfo icebergRestConfigModel = + (IcebergRestConnectionConfigInfo) connectionConfigurationModel; + PolarisRestAuthenticationInfo restAuthenticationInfo = + PolarisRestAuthenticationInfo.fromRestAuthenticationInfoModel( + icebergRestConfigModel.getRestAuthentication()); + config = + new IcebergRestConnectionConfigurationInfo( + ConnectionType.ICEBERG_REST, + icebergRestConfigModel.getRemoteUri(), + icebergRestConfigModel.getRemoteCatalogName(), + restAuthenticationInfo); + break; + default: + throw new IllegalStateException( + "Unsupported connection type: " + connectionConfigurationModel.getConnectionType()); + } + internalProperties.put( + PolarisEntityConstants.getConnectionConfigInfoPropertyName(), config.serialize()); + } + return this; + } + @Override public CatalogEntity build() { return new CatalogEntity(buildBase()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityConstants.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityConstants.java index 26d3c09a73..4452a90200 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityConstants.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityConstants.java @@ -52,6 +52,9 @@ public class PolarisEntityConstants { private static final String STORAGE_INTEGRATION_IDENTIFIER_PROPERTY_NAME = "storage_integration_identifier"; + private static final String CONNECTION_CONFIGURATION_INFO_PROPERTY_NAME = + "connection_configuration_info"; + private static final String PRINCIPAL_TYPE_NAME = "principal_type_name"; public static final String PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE = @@ -104,6 +107,10 @@ public static String getStorageConfigInfoPropertyName() { return STORAGE_CONFIGURATION_INFO_PROPERTY_NAME; } + public static String getConnectionConfigInfoPropertyName() { + return CONNECTION_CONFIGURATION_INFO_PROPERTY_NAME; + } + public static String getPolarisStorageIntegrationNameFormat() { return POLARIS_STORAGE_INT_NAME_FORMAT; } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index bcb377ebe7..d5faa66ebe 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -19,7 +19,6 @@ package org.apache.polaris.service.catalog.iceberg; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import jakarta.ws.rs.core.SecurityContext; import java.io.Closeable; @@ -56,7 +55,6 @@ import org.apache.iceberg.rest.CatalogHandlers; import org.apache.iceberg.rest.HTTPClient; import org.apache.iceberg.rest.RESTCatalog; -import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; @@ -76,6 +74,7 @@ import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.connection.PolarisConnectionConfigurationInfo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; @@ -161,37 +160,34 @@ public static boolean isCreate(UpdateTableRequest request) { protected void initializeCatalog() { CatalogEntity resolvedCatalogEntity = CatalogEntity.of(resolutionManifest.getResolvedReferenceCatalogEntity().getRawLeafEntity()); - if (resolvedCatalogEntity.getConnectionRemoteUri() != null) { + PolarisConnectionConfigurationInfo connectionConfigurationInfo = + resolvedCatalogEntity.getConnectionConfigurationInfo(); + if (connectionConfigurationInfo != null) { LOGGER .atInfo() - .addKeyValue("remoteUrl", resolvedCatalogEntity.getConnectionRemoteUri()) + .addKeyValue("remoteUrl", connectionConfigurationInfo.getRemoteUri()) .log("Initializing federated catalog"); - SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); - RESTCatalog restCatalog = - new RESTCatalog( - context, - (config) -> - HTTPClient.builder(config) - .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) - .build()); - - ImmutableMap.Builder propertiesBuilder = - ImmutableMap.builder() - .put( - org.apache.iceberg.CatalogProperties.URI, - resolvedCatalogEntity.getConnectionRemoteUri()) - .put( - OAuth2Properties.CREDENTIAL, - resolvedCatalogEntity.getConnectionClientId() - + ":" - + resolvedCatalogEntity.getConnectionClientSecret()) - .put(OAuth2Properties.SCOPE, resolvedCatalogEntity.getConnectionScopes()) - .put("warehouse", resolvedCatalogEntity.getConnectionCatalogName()); - - restCatalog.initialize( - resolvedCatalogEntity.getConnectionCatalogName(), propertiesBuilder.buildKeepingLast()); - this.baseCatalog = restCatalog; + Catalog federatedCatalog; + switch (connectionConfigurationInfo.getConnectionType()) { + case ICEBERG_REST: + SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); + federatedCatalog = + new RESTCatalog( + context, + (config) -> + HTTPClient.builder(config) + .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) + .build()); + break; + default: + throw new UnsupportedOperationException( + "Connection type not supported: " + connectionConfigurationInfo.getConnectionType()); + } + federatedCatalog.initialize( + resolvedCatalogEntity.getConnectionCatalogName(), + connectionConfigurationInfo.asIcebergCatalogProperties()); + this.baseCatalog = federatedCatalog; } else { LOGGER.atInfo().log("Initializing non-federated catalog"); this.baseCatalog = From 205ef1ed01499da4f2c599e22870c9d95f6adfdd Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Wed, 19 Mar 2025 21:29:37 +0000 Subject: [PATCH 04/17] Apply the same spec renames to the internal ConnectionConfiguration representations. --- ...cebergRestConnectionConfigurationInfo.java | 21 +++++----- ...a => PolarisAuthenticationParameters.java} | 41 +++++++++---------- ...olarisBearerAuthenticationParameters.java} | 17 ++++---- .../PolarisConnectionConfigurationInfo.java | 26 ++++++------ ...arisOAuthClientCredentialsParameters.java} | 17 ++++---- .../polaris/core/entity/CatalogEntity.java | 16 ++++---- ...olarisConnectionConfigurationInfoTest.java | 8 ++-- .../iceberg/IcebergCatalogHandler.java | 2 +- 8 files changed, 73 insertions(+), 75 deletions(-) rename polaris-core/src/main/java/org/apache/polaris/core/connection/{PolarisRestAuthenticationInfo.java => PolarisAuthenticationParameters.java} (61%) rename polaris-core/src/main/java/org/apache/polaris/core/connection/{PolarisBearerRestAuthenticationInfo.java => PolarisBearerAuthenticationParameters.java} (76%) rename polaris-core/src/main/java/org/apache/polaris/core/connection/{PolarisOauthRestAuthenticationInfo.java => PolarisOAuthClientCredentialsParameters.java} (87%) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java index d1900722ed..38e19f9417 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java @@ -27,24 +27,23 @@ import org.apache.iceberg.CatalogProperties; import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; -import org.jetbrains.annotations.NotNull; public class IcebergRestConnectionConfigurationInfo extends PolarisConnectionConfigurationInfo implements IcebergCatalogPropertiesProvider { private final String remoteCatalogName; - private final PolarisRestAuthenticationInfo restAuthentication; + private final PolarisAuthenticationParameters restAuthentication; public IcebergRestConnectionConfigurationInfo( @JsonProperty(value = "connectionType", required = true) @Nonnull ConnectionType connectionType, - @JsonProperty(value = "remoteUri", required = true) @Nonnull String remoteUri, + @JsonProperty(value = "uri", required = true) @Nonnull String uri, @JsonProperty(value = "remoteCatalogName", required = false) @Nullable String remoteCatalogName, @JsonProperty(value = "restAuthentication", required = false) @Nonnull - PolarisRestAuthenticationInfo restAuthentication) { - super(connectionType, remoteUri); + PolarisAuthenticationParameters restAuthentication) { + super(connectionType, uri); this.remoteCatalogName = remoteCatalogName; this.restAuthentication = restAuthentication; } @@ -53,14 +52,14 @@ public String getRemoteCatalogName() { return remoteCatalogName; } - public PolarisRestAuthenticationInfo getRestAuthentication() { + public PolarisAuthenticationParameters getRestAuthentication() { return restAuthentication; } @Override - public @NotNull Map asIcebergCatalogProperties() { + public @Nonnull Map asIcebergCatalogProperties() { HashMap properties = new HashMap<>(); - properties.put(CatalogProperties.URI, getRemoteUri()); + properties.put(CatalogProperties.URI, getUri()); if (getRemoteCatalogName() != null) { properties.put(CatalogProperties.WAREHOUSE_LOCATION, getRemoteCatalogName()); } @@ -72,9 +71,9 @@ public PolarisRestAuthenticationInfo getRestAuthentication() { public ConnectionConfigInfo asConnectionConfigInfoModel() { return IcebergRestConnectionConfigInfo.builder() .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) - .setRemoteUri(getRemoteUri()) + .setUri(getUri()) .setRemoteCatalogName(getRemoteCatalogName()) - .setRestAuthentication(restAuthentication.asRestAuthenticationInfoModel()) + .setRestAuthentication(restAuthentication.asAuthenticationParametersModel()) .build(); } @@ -83,7 +82,7 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("connectionType", getConnectionType()) .add("connectionType", getConnectionType().name()) - .add("remoteUri", getRemoteUri()) + .add("uri", getUri()) .add("remoteCatalogName", getRemoteCatalogName()) .add("restAuthentication", getRestAuthentication().toString()) .toString(); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java similarity index 61% rename from polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java rename to polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java index a1a48cad04..e7b947c445 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisRestAuthenticationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java @@ -23,22 +23,21 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import jakarta.annotation.Nonnull; import java.util.Map; -import org.apache.polaris.core.admin.model.BearerRestAuthenticationInfo; -import org.apache.polaris.core.admin.model.OauthRestAuthenticationInfo; -import org.apache.polaris.core.admin.model.RestAuthenticationInfo; -import org.jetbrains.annotations.NotNull; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; +import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "restAuthenticationType", visible = true) @JsonSubTypes({ - @JsonSubTypes.Type(value = PolarisOauthRestAuthenticationInfo.class, name = "OAUTH"), - @JsonSubTypes.Type(value = PolarisBearerRestAuthenticationInfo.class, name = "BEARER"), + @JsonSubTypes.Type(value = PolarisOAuthClientCredentialsParameters.class, name = "OAUTH"), + @JsonSubTypes.Type(value = PolarisBearerAuthenticationParameters.class, name = "BEARER"), }) -public abstract class PolarisRestAuthenticationInfo implements IcebergCatalogPropertiesProvider { +public abstract class PolarisAuthenticationParameters implements IcebergCatalogPropertiesProvider { @JsonProperty(value = "restAuthenticationType") private final RestAuthenticationType restAuthenticationType; - public PolarisRestAuthenticationInfo( + public PolarisAuthenticationParameters( @JsonProperty(value = "restAuthenticationType", required = true) @Nonnull RestAuthenticationType restAuthenticationType) { this.restAuthenticationType = restAuthenticationType; @@ -49,19 +48,19 @@ public PolarisRestAuthenticationInfo( } @Override - public abstract @NotNull Map asIcebergCatalogProperties(); + public abstract @Nonnull Map asIcebergCatalogProperties(); - public abstract RestAuthenticationInfo asRestAuthenticationInfoModel(); + public abstract AuthenticationParameters asAuthenticationParametersModel(); - public static PolarisRestAuthenticationInfo fromRestAuthenticationInfoModel( - RestAuthenticationInfo restAuthenticationInfo) { - PolarisRestAuthenticationInfo config = null; - switch (restAuthenticationInfo.getRestAuthenticationType()) { + public static PolarisAuthenticationParameters fromAuthenticationParametersModel( + AuthenticationParameters restAuthenticationParameters) { + PolarisAuthenticationParameters config = null; + switch (restAuthenticationParameters.getRestAuthenticationType()) { case OAUTH: - OauthRestAuthenticationInfo oauthRestAuthenticationModel = - (OauthRestAuthenticationInfo) restAuthenticationInfo; + OAuthClientCredentialsParameters oauthRestAuthenticationModel = + (OAuthClientCredentialsParameters) restAuthenticationParameters; config = - new PolarisOauthRestAuthenticationInfo( + new PolarisOAuthClientCredentialsParameters( RestAuthenticationType.OAUTH, oauthRestAuthenticationModel.getTokenUri(), oauthRestAuthenticationModel.getClientId(), @@ -69,16 +68,16 @@ public static PolarisRestAuthenticationInfo fromRestAuthenticationInfoModel( oauthRestAuthenticationModel.getScopes()); break; case BEARER: - BearerRestAuthenticationInfo bearerRestAuthenticationModel = - (BearerRestAuthenticationInfo) restAuthenticationInfo; + BearerAuthenticationParameters bearerRestAuthenticationModel = + (BearerAuthenticationParameters) restAuthenticationParameters; config = - new PolarisBearerRestAuthenticationInfo( + new PolarisBearerAuthenticationParameters( RestAuthenticationType.BEARER, bearerRestAuthenticationModel.getBearerToken()); break; default: throw new IllegalStateException( "Unsupported authentication type: " - + restAuthenticationInfo.getRestAuthenticationType()); + + restAuthenticationParameters.getRestAuthenticationType()); } return config; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java similarity index 76% rename from polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java rename to polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java index acd339e7d6..6007e9f32a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerRestAuthenticationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java @@ -23,16 +23,15 @@ import jakarta.annotation.Nonnull; import java.util.Map; import org.apache.iceberg.rest.auth.OAuth2Properties; -import org.apache.polaris.core.admin.model.BearerRestAuthenticationInfo; -import org.apache.polaris.core.admin.model.RestAuthenticationInfo; -import org.jetbrains.annotations.NotNull; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; -public class PolarisBearerRestAuthenticationInfo extends PolarisRestAuthenticationInfo { +public class PolarisBearerAuthenticationParameters extends PolarisAuthenticationParameters { @JsonProperty(value = "bearerToken") private final String bearerToken; - public PolarisBearerRestAuthenticationInfo( + public PolarisBearerAuthenticationParameters( @JsonProperty(value = "restAuthenticationType", required = true) @Nonnull RestAuthenticationType restAuthenticationType, @JsonProperty(value = "bearerToken", required = true) @Nonnull String bearerToken) { @@ -45,15 +44,15 @@ public PolarisBearerRestAuthenticationInfo( } @Override - public @NotNull Map asIcebergCatalogProperties() { + public @Nonnull Map asIcebergCatalogProperties() { return Map.of(OAuth2Properties.TOKEN, getBearerToken()); } @Override - public RestAuthenticationInfo asRestAuthenticationInfoModel() { + public AuthenticationParameters asAuthenticationParametersModel() { // TODO: redact secrets from the model - return BearerRestAuthenticationInfo.builder() - .setRestAuthenticationType(RestAuthenticationInfo.RestAuthenticationTypeEnum.BEARER) + return BearerAuthenticationParameters.builder() + .setRestAuthenticationType(AuthenticationParameters.RestAuthenticationTypeEnum.BEARER) .setBearerToken(getBearerToken()) .build(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java index 83d57a5452..a6ba2d5f48 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java @@ -47,21 +47,21 @@ public abstract class PolarisConnectionConfigurationInfo private final ConnectionType connectionType; // The URI of the remote catalog - private final String remoteUri; + private final String uri; public PolarisConnectionConfigurationInfo( @JsonProperty(value = "connectionType", required = true) @Nonnull ConnectionType connectionType, - @JsonProperty(value = "remoteUri", required = true) @Nonnull String remoteUri) { - this(connectionType, remoteUri, true); + @JsonProperty(value = "uri", required = true) @Nonnull String uri) { + this(connectionType, uri, true); } protected PolarisConnectionConfigurationInfo( - ConnectionType connectionType, String remoteUri, boolean validateRemoteUri) { + ConnectionType connectionType, String uri, boolean validateUri) { this.connectionType = connectionType; - this.remoteUri = remoteUri; - if (validateRemoteUri) { - validateRemoteUri(remoteUri); + this.uri = uri; + if (validateUri) { + validateUri(uri); } } @@ -69,8 +69,8 @@ public ConnectionType getConnectionType() { return connectionType; } - public String getRemoteUri() { - return remoteUri; + public String getUri() { + return uri; } private static final ObjectMapper DEFAULT_MAPPER; @@ -101,12 +101,12 @@ public static PolarisConnectionConfigurationInfo deserialize( } /** Validates the remote URI. */ - protected void validateRemoteUri(String remoteUri) { + protected void validateUri(String uri) { try { - URI uri = URI.create(remoteUri); - URL url = uri.toURL(); + URI uriObj = URI.create(uri); + URL url = uriObj.toURL(); } catch (IllegalArgumentException | MalformedURLException e) { - throw new IllegalArgumentException("Invalid remote URI: " + remoteUri, e); + throw new IllegalArgumentException("Invalid remote URI: " + uri, e); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java similarity index 87% rename from polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java rename to polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java index 472bfc2349..b515bc3404 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOauthRestAuthenticationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java @@ -33,11 +33,10 @@ import java.util.Objects; import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.auth.OAuth2Util; -import org.apache.polaris.core.admin.model.OauthRestAuthenticationInfo; -import org.apache.polaris.core.admin.model.RestAuthenticationInfo; -import org.jetbrains.annotations.NotNull; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; -public class PolarisOauthRestAuthenticationInfo extends PolarisRestAuthenticationInfo { +public class PolarisOAuthClientCredentialsParameters extends PolarisAuthenticationParameters { private static final Joiner COLON_JOINER = Joiner.on(":"); @@ -53,7 +52,7 @@ public class PolarisOauthRestAuthenticationInfo extends PolarisRestAuthenticatio @JsonProperty(value = "scopes") private final List scopes; - public PolarisOauthRestAuthenticationInfo( + public PolarisOAuthClientCredentialsParameters( @JsonProperty(value = "restAuthenticationType", required = true) @Nonnull RestAuthenticationType restAuthenticationType, @JsonProperty(value = "tokenUri", required = false) @Nullable String tokenUri, @@ -98,7 +97,7 @@ public PolarisOauthRestAuthenticationInfo( } @Override - public @NotNull Map asIcebergCatalogProperties() { + public @Nonnull Map asIcebergCatalogProperties() { HashMap properties = new HashMap<>(); if (getTokenUri() != null) { properties.put(OAuth2Properties.OAUTH2_SERVER_URI, getTokenUri()); @@ -109,10 +108,10 @@ public PolarisOauthRestAuthenticationInfo( } @Override - public RestAuthenticationInfo asRestAuthenticationInfoModel() { + public AuthenticationParameters asAuthenticationParametersModel() { // TODO: redact secrets from the model - return OauthRestAuthenticationInfo.builder() - .setRestAuthenticationType(RestAuthenticationInfo.RestAuthenticationTypeEnum.OAUTH) + return OAuthClientCredentialsParameters.builder() + .setRestAuthenticationType(AuthenticationParameters.RestAuthenticationTypeEnum.OAUTH) .setTokenUri(getTokenUri()) .setClientId(getClientId()) .setClientSecret(getClientSecret()) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index 31d41b060a..1e269a7bed 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -35,16 +35,18 @@ import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; +import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.config.BehaviorChangeConfiguration; import org.apache.polaris.core.connection.ConnectionType; import org.apache.polaris.core.connection.IcebergRestConnectionConfigurationInfo; +import org.apache.polaris.core.connection.PolarisAuthenticationParameters; import org.apache.polaris.core.connection.PolarisConnectionConfigurationInfo; -import org.apache.polaris.core.connection.PolarisRestAuthenticationInfo; import org.apache.polaris.core.storage.FileStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; @@ -71,7 +73,7 @@ public class CatalogEntity extends PolarisEntity { "replace-new-location-prefix-with-catalog-default"; // TODO: Refactor all these into ConnectionConfigurationInfo - public static final String CONNECTION_REMOTE_URI_KEY = "connection.remoteUri"; + public static final String CONNECTION_REMOTE_URI_KEY = "connection.uri"; public static final String CONNECTION_CLIENT_ID_KEY = "connection.clientId"; public static final String CONNECTION_CLIENT_SECRET_KEY = "connection.clientSecret"; public static final String CONNECTION_CATALOG_NAME_KEY = "connection.catalogName"; @@ -230,7 +232,7 @@ public PolarisConnectionConfigurationInfo getConnectionConfigurationInfo() { return null; } - public String getConnectionRemoteUri() { + public String getConnectionUri() { // TODO: Refactor this to use new ConnectionConfigurationInfo return getPropertiesAsMap().get(CONNECTION_REMOTE_URI_KEY); } @@ -363,15 +365,15 @@ public Builder setConnectionConfigurationInfo( case ICEBERG_REST: IcebergRestConnectionConfigInfo icebergRestConfigModel = (IcebergRestConnectionConfigInfo) connectionConfigurationModel; - PolarisRestAuthenticationInfo restAuthenticationInfo = - PolarisRestAuthenticationInfo.fromRestAuthenticationInfoModel( + PolarisAuthenticationParameters restAuthenticationParameters = + PolarisAuthenticationParameters.fromAuthenticationParametersModel( icebergRestConfigModel.getRestAuthentication()); config = new IcebergRestConnectionConfigurationInfo( ConnectionType.ICEBERG_REST, - icebergRestConfigModel.getRemoteUri(), + icebergRestConfigModel.getUri(), icebergRestConfigModel.getRemoteCatalogName(), - restAuthenticationInfo); + restAuthenticationParameters); break; default: throw new IllegalStateException( diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java index 38babf2d09..23a249710c 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java @@ -31,12 +31,12 @@ public class PolarisConnectionConfigurationInfoTest { ObjectMapper objectMapper = new ObjectMapper(); @Test - void testOauthRestAuthenticationInfo() throws JsonProcessingException { + void testOAuthClientCredentialsParameters() throws JsonProcessingException { String json = "" + "{" + " \"connectionType\": \"ICEBERG_REST\"," - + " \"remoteUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + " \"remoteCatalogName\": \"my-catalog\"," + " \"restAuthentication\": {" + " \"restAuthenticationType\": \"OAUTH\"," @@ -56,12 +56,12 @@ void testOauthRestAuthenticationInfo() throws JsonProcessingException { } @Test - void testBearerRestAuthenticationInfo() throws JsonProcessingException { + void testBearerAuthenticationParameters() throws JsonProcessingException { String json = "" + "{" + " \"connectionType\": \"ICEBERG_REST\"," - + " \"remoteUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + " \"remoteCatalogName\": \"my-catalog\"," + " \"restAuthentication\": {" + " \"restAuthenticationType\": \"BEARER\"," diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index d5faa66ebe..673f1396fc 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -165,7 +165,7 @@ protected void initializeCatalog() { if (connectionConfigurationInfo != null) { LOGGER .atInfo() - .addKeyValue("remoteUrl", connectionConfigurationInfo.getRemoteUri()) + .addKeyValue("remoteUrl", connectionConfigurationInfo.getUri()) .log("Initializing federated catalog"); Catalog federatedCatalog; From c8484e0dfccaca974174e323680194acfdae0f1c Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Wed, 19 Mar 2025 21:49:57 +0000 Subject: [PATCH 05/17] Manually pick @XJDKC fixes for integration tests and omittign secrets in response objects --- .../service/it/test/PolarisApplicationIntegrationTest.java | 4 ++-- .../core/connection/PolarisAuthenticationParameters.java | 4 ---- .../connection/PolarisBearerAuthenticationParameters.java | 1 - .../connection/PolarisOAuthClientCredentialsParameters.java | 1 - 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index 99e736f798..2788567da7 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -378,7 +378,7 @@ public void testIcebergCreateTablesInExternalCatalog() throws IOException { .withPartitionSpec(PartitionSpec.unpartitioned()) .create()) .isInstanceOf(BadRequestException.class) - .hasMessage("Malformed request: Cannot create table on external catalogs."); + .hasMessage("Malformed request: Cannot create table on static-facade external catalogs."); } } @@ -515,7 +515,7 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException { 10L)) .commit()) .isInstanceOf(BadRequestException.class) - .hasMessage("Malformed request: Cannot update table on external catalogs."); + .hasMessage("Malformed request: Cannot update table on static-facade external catalogs."); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java index e7b947c445..9eef670140 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import jakarta.annotation.Nonnull; -import java.util.Map; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; @@ -47,9 +46,6 @@ public PolarisAuthenticationParameters( return restAuthenticationType; } - @Override - public abstract @Nonnull Map asIcebergCatalogProperties(); - public abstract AuthenticationParameters asAuthenticationParametersModel(); public static PolarisAuthenticationParameters fromAuthenticationParametersModel( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java index 6007e9f32a..0b75d0249a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java @@ -61,7 +61,6 @@ public AuthenticationParameters asAuthenticationParametersModel() { public String toString() { return MoreObjects.toStringHelper(this) .add("authenticationType", getRestAuthenticationType()) - .add("bearerToken", getBearerToken()) .toString(); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java index b515bc3404..3e2bcee1a0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java @@ -114,7 +114,6 @@ public AuthenticationParameters asAuthenticationParametersModel() { .setRestAuthenticationType(AuthenticationParameters.RestAuthenticationTypeEnum.OAUTH) .setTokenUri(getTokenUri()) .setClientId(getClientId()) - .setClientSecret(getClientSecret()) .setScopes(getScopes()) .build(); } From acb5becd1666f08e6279c51d0bccfb3af39e0fa5 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Thu, 20 Mar 2025 07:16:23 +0000 Subject: [PATCH 06/17] Fix internal connection structs with updated naming from spec PR --- ...ationType.java => AuthenticationType.java} | 2 +- ...cebergRestConnectionConfigurationInfo.java | 18 ++++---- .../PolarisAuthenticationParameters.java | 43 +++++++++---------- ...PolarisBearerAuthenticationParameters.java | 10 ++--- ...larisOAuthClientCredentialsParameters.java | 8 ++-- .../polaris/core/entity/CatalogEntity.java | 6 +-- ...olarisConnectionConfigurationInfoTest.java | 8 ++-- 7 files changed, 47 insertions(+), 48 deletions(-) rename polaris-core/src/main/java/org/apache/polaris/core/connection/{RestAuthenticationType.java => AuthenticationType.java} (95%) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/RestAuthenticationType.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java similarity index 95% rename from polaris-core/src/main/java/org/apache/polaris/core/connection/RestAuthenticationType.java rename to polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java index dcbb623059..03608bba88 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/RestAuthenticationType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java @@ -18,7 +18,7 @@ */ package org.apache.polaris.core.connection; -public enum RestAuthenticationType { +public enum AuthenticationType { OAUTH, BEARER } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java index 38e19f9417..35609c5fb7 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java @@ -33,7 +33,7 @@ public class IcebergRestConnectionConfigurationInfo extends PolarisConnectionCon private final String remoteCatalogName; - private final PolarisAuthenticationParameters restAuthentication; + private final PolarisAuthenticationParameters authenticationParameters; public IcebergRestConnectionConfigurationInfo( @JsonProperty(value = "connectionType", required = true) @Nonnull @@ -41,19 +41,19 @@ public IcebergRestConnectionConfigurationInfo( @JsonProperty(value = "uri", required = true) @Nonnull String uri, @JsonProperty(value = "remoteCatalogName", required = false) @Nullable String remoteCatalogName, - @JsonProperty(value = "restAuthentication", required = false) @Nonnull - PolarisAuthenticationParameters restAuthentication) { + @JsonProperty(value = "authenticationParameters", required = false) @Nonnull + PolarisAuthenticationParameters authenticationParameters) { super(connectionType, uri); this.remoteCatalogName = remoteCatalogName; - this.restAuthentication = restAuthentication; + this.authenticationParameters = authenticationParameters; } public String getRemoteCatalogName() { return remoteCatalogName; } - public PolarisAuthenticationParameters getRestAuthentication() { - return restAuthentication; + public PolarisAuthenticationParameters getAuthenticationParameters() { + return authenticationParameters; } @Override @@ -63,7 +63,7 @@ public PolarisAuthenticationParameters getRestAuthentication() { if (getRemoteCatalogName() != null) { properties.put(CatalogProperties.WAREHOUSE_LOCATION, getRemoteCatalogName()); } - properties.putAll(restAuthentication.asIcebergCatalogProperties()); + properties.putAll(authenticationParameters.asIcebergCatalogProperties()); return properties; } @@ -73,7 +73,7 @@ public ConnectionConfigInfo asConnectionConfigInfoModel() { .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) .setUri(getUri()) .setRemoteCatalogName(getRemoteCatalogName()) - .setRestAuthentication(restAuthentication.asAuthenticationParametersModel()) + .setAuthenticationParameters(authenticationParameters.asAuthenticationParametersModel()) .build(); } @@ -84,7 +84,7 @@ public String toString() { .add("connectionType", getConnectionType().name()) .add("uri", getUri()) .add("remoteCatalogName", getRemoteCatalogName()) - .add("restAuthentication", getRestAuthentication().toString()) + .add("authenticationParameters", getAuthenticationParameters().toString()) .toString(); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java index 9eef670140..adf49edd81 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java @@ -26,54 +26,53 @@ import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "restAuthenticationType", visible = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "authenticationType", visible = true) @JsonSubTypes({ @JsonSubTypes.Type(value = PolarisOAuthClientCredentialsParameters.class, name = "OAUTH"), @JsonSubTypes.Type(value = PolarisBearerAuthenticationParameters.class, name = "BEARER"), }) public abstract class PolarisAuthenticationParameters implements IcebergCatalogPropertiesProvider { - @JsonProperty(value = "restAuthenticationType") - private final RestAuthenticationType restAuthenticationType; + @JsonProperty(value = "authenticationType") + private final AuthenticationType authenticationType; public PolarisAuthenticationParameters( - @JsonProperty(value = "restAuthenticationType", required = true) @Nonnull - RestAuthenticationType restAuthenticationType) { - this.restAuthenticationType = restAuthenticationType; + @JsonProperty(value = "authenticationType", required = true) @Nonnull + AuthenticationType authenticationType) { + this.authenticationType = authenticationType; } - public @Nonnull RestAuthenticationType getRestAuthenticationType() { - return restAuthenticationType; + public @Nonnull AuthenticationType getAuthenticationType() { + return authenticationType; } public abstract AuthenticationParameters asAuthenticationParametersModel(); public static PolarisAuthenticationParameters fromAuthenticationParametersModel( - AuthenticationParameters restAuthenticationParameters) { + AuthenticationParameters authenticationParameters) { PolarisAuthenticationParameters config = null; - switch (restAuthenticationParameters.getRestAuthenticationType()) { + switch (authenticationParameters.getAuthenticationType()) { case OAUTH: - OAuthClientCredentialsParameters oauthRestAuthenticationModel = - (OAuthClientCredentialsParameters) restAuthenticationParameters; + OAuthClientCredentialsParameters oauthClientCredentialsModel = + (OAuthClientCredentialsParameters) authenticationParameters; config = new PolarisOAuthClientCredentialsParameters( - RestAuthenticationType.OAUTH, - oauthRestAuthenticationModel.getTokenUri(), - oauthRestAuthenticationModel.getClientId(), - oauthRestAuthenticationModel.getClientSecret(), - oauthRestAuthenticationModel.getScopes()); + AuthenticationType.OAUTH, + oauthClientCredentialsModel.getTokenUri(), + oauthClientCredentialsModel.getClientId(), + oauthClientCredentialsModel.getClientSecret(), + oauthClientCredentialsModel.getScopes()); break; case BEARER: - BearerAuthenticationParameters bearerRestAuthenticationModel = - (BearerAuthenticationParameters) restAuthenticationParameters; + BearerAuthenticationParameters bearerAuthenticationParametersModel = + (BearerAuthenticationParameters) authenticationParameters; config = new PolarisBearerAuthenticationParameters( - RestAuthenticationType.BEARER, bearerRestAuthenticationModel.getBearerToken()); + AuthenticationType.BEARER, bearerAuthenticationParametersModel.getBearerToken()); break; default: throw new IllegalStateException( - "Unsupported authentication type: " - + restAuthenticationParameters.getRestAuthenticationType()); + "Unsupported authentication type: " + authenticationParameters.getAuthenticationType()); } return config; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java index 0b75d0249a..2c4dbea7ca 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java @@ -32,10 +32,10 @@ public class PolarisBearerAuthenticationParameters extends PolarisAuthentication private final String bearerToken; public PolarisBearerAuthenticationParameters( - @JsonProperty(value = "restAuthenticationType", required = true) @Nonnull - RestAuthenticationType restAuthenticationType, + @JsonProperty(value = "authenticationType", required = true) @Nonnull + AuthenticationType authenticationType, @JsonProperty(value = "bearerToken", required = true) @Nonnull String bearerToken) { - super(restAuthenticationType); + super(authenticationType); this.bearerToken = bearerToken; } @@ -52,7 +52,7 @@ public PolarisBearerAuthenticationParameters( public AuthenticationParameters asAuthenticationParametersModel() { // TODO: redact secrets from the model return BearerAuthenticationParameters.builder() - .setRestAuthenticationType(AuthenticationParameters.RestAuthenticationTypeEnum.BEARER) + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.BEARER) .setBearerToken(getBearerToken()) .build(); } @@ -60,7 +60,7 @@ public AuthenticationParameters asAuthenticationParametersModel() { @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("authenticationType", getRestAuthenticationType()) + .add("authenticationType", getAuthenticationType()) .toString(); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java index 3e2bcee1a0..664266d4a3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java @@ -53,13 +53,13 @@ public class PolarisOAuthClientCredentialsParameters extends PolarisAuthenticati private final List scopes; public PolarisOAuthClientCredentialsParameters( - @JsonProperty(value = "restAuthenticationType", required = true) @Nonnull - RestAuthenticationType restAuthenticationType, + @JsonProperty(value = "authenticationType", required = true) @Nonnull + AuthenticationType authenticationType, @JsonProperty(value = "tokenUri", required = false) @Nullable String tokenUri, @JsonProperty(value = "clientId", required = true) @Nonnull String clientId, @JsonProperty(value = "clientSecret", required = true) @Nonnull String clientSecret, @JsonProperty(value = "scopes", required = false) @Nullable List scopes) { - super(restAuthenticationType); + super(authenticationType); this.tokenUri = tokenUri; this.clientId = clientId; @@ -111,7 +111,7 @@ public PolarisOAuthClientCredentialsParameters( public AuthenticationParameters asAuthenticationParametersModel() { // TODO: redact secrets from the model return OAuthClientCredentialsParameters.builder() - .setRestAuthenticationType(AuthenticationParameters.RestAuthenticationTypeEnum.OAUTH) + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.OAUTH) .setTokenUri(getTokenUri()) .setClientId(getClientId()) .setScopes(getScopes()) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index 1e269a7bed..3dabbfb8e0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -365,15 +365,15 @@ public Builder setConnectionConfigurationInfo( case ICEBERG_REST: IcebergRestConnectionConfigInfo icebergRestConfigModel = (IcebergRestConnectionConfigInfo) connectionConfigurationModel; - PolarisAuthenticationParameters restAuthenticationParameters = + PolarisAuthenticationParameters authenticationParameters = PolarisAuthenticationParameters.fromAuthenticationParametersModel( - icebergRestConfigModel.getRestAuthentication()); + icebergRestConfigModel.getAuthenticationParameters()); config = new IcebergRestConnectionConfigurationInfo( ConnectionType.ICEBERG_REST, icebergRestConfigModel.getUri(), icebergRestConfigModel.getRemoteCatalogName(), - restAuthenticationParameters); + authenticationParameters); break; default: throw new IllegalStateException( diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java index 23a249710c..d9e0febebe 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java @@ -38,8 +38,8 @@ void testOAuthClientCredentialsParameters() throws JsonProcessingException { + " \"connectionType\": \"ICEBERG_REST\"," + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + " \"remoteCatalogName\": \"my-catalog\"," - + " \"restAuthentication\": {" - + " \"restAuthenticationType\": \"OAUTH\"," + + " \"authenticationParameters\": {" + + " \"authenticationType\": \"OAUTH\"," + " \"tokenUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog/v1/oauth/tokens\"," + " \"clientId\": \"client-id\"," + " \"clientSecret\": \"client-secret\"," @@ -63,8 +63,8 @@ void testBearerAuthenticationParameters() throws JsonProcessingException { + " \"connectionType\": \"ICEBERG_REST\"," + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + " \"remoteCatalogName\": \"my-catalog\"," - + " \"restAuthentication\": {" - + " \"restAuthenticationType\": \"BEARER\"," + + " \"authenticationParameters\": {" + + " \"authenticationType\": \"BEARER\"," + " \"bearerToken\": \"bearer-token\"" + " }" + "}"; From 300553f5c4a408a3a43f4d7ae454eba8d8f7ed14 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Thu, 27 Mar 2025 01:59:04 +0000 Subject: [PATCH 07/17] Push CreateCatalogRequest down to PolarisAdminService::createCatalog just like UpdateCatalogRequest in updateCatalog. This is needed if we're going to make PolarisAdminService handle secrets management without ever putting the secrets into a CatalogEntity. --- .../admin/PolarisAdminServiceAuthzTest.java | 15 +++-- .../quarkus/admin/PolarisAuthzTestBase.java | 15 +++-- .../catalog/GenericTableCatalogTest.java | 24 ++++--- .../IcebergCatalogHandlerAuthzTest.java | 15 +++-- .../quarkus/catalog/IcebergCatalogTest.java | 67 +++++++++++-------- .../catalog/IcebergCatalogViewTest.java | 26 ++++--- .../service/admin/PolarisAdminService.java | 5 +- .../service/admin/PolarisServiceImpl.java | 4 +- 8 files changed, 101 insertions(+), 70 deletions(-) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java index 5c055eec47..7199f4fa7e 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.UpdateCatalogRequest; import org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest; import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; @@ -130,13 +131,14 @@ public void testCreateCatalogSufficientPrivileges() { PRINCIPAL_ROLE2, PolarisPrivilege.CATALOG_DROP)) .isTrue(); final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); + final CreateCatalogRequest createRequest = new CreateCatalogRequest(newCatalog.asCatalog()); doTestSufficientPrivileges( List.of( PolarisPrivilege.SERVICE_MANAGE_ACCESS, PolarisPrivilege.CATALOG_CREATE, PolarisPrivilege.CATALOG_FULL_METADATA), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(newCatalog), + () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(createRequest), () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).deleteCatalog(newCatalog.getName()), (privilege) -> adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), @@ -148,6 +150,7 @@ public void testCreateCatalogSufficientPrivileges() { @Test public void testCreateCatalogInsufficientPrivileges() { final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); + final CreateCatalogRequest createRequest = new CreateCatalogRequest(newCatalog.asCatalog()); doTestInsufficientPrivileges( List.of( @@ -164,7 +167,7 @@ public void testCreateCatalogInsufficientPrivileges() { PolarisPrivilege.CATALOG_MANAGE_METADATA, PolarisPrivilege.CATALOG_MANAGE_CONTENT, PolarisPrivilege.CATALOG_MANAGE_ACCESS), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(newCatalog), + () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).createCatalog(createRequest), (privilege) -> adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), (privilege) -> @@ -283,7 +286,8 @@ public void testDeleteCatalogSufficientPrivileges() { PRINCIPAL_ROLE2, PolarisPrivilege.CATALOG_CREATE)) .isTrue(); final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); - adminService.createCatalog(newCatalog); + final CreateCatalogRequest createRequest = new CreateCatalogRequest(newCatalog.asCatalog()); + adminService.createCatalog(createRequest); doTestSufficientPrivileges( List.of( @@ -291,7 +295,7 @@ public void testDeleteCatalogSufficientPrivileges() { PolarisPrivilege.CATALOG_DROP, PolarisPrivilege.CATALOG_FULL_METADATA), () -> newTestAdminService(Set.of(PRINCIPAL_ROLE1)).deleteCatalog(newCatalog.getName()), - () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).createCatalog(newCatalog), + () -> newTestAdminService(Set.of(PRINCIPAL_ROLE2)).createCatalog(createRequest), (privilege) -> adminService.grantPrivilegeOnRootContainerToPrincipalRole(PRINCIPAL_ROLE1, privilege), (privilege) -> @@ -302,7 +306,8 @@ public void testDeleteCatalogSufficientPrivileges() { @Test public void testDeleteCatalogInsufficientPrivileges() { final CatalogEntity newCatalog = new CatalogEntity.Builder().setName("new_catalog").build(); - adminService.createCatalog(newCatalog); + final CreateCatalogRequest createRequest = new CreateCatalogRequest(newCatalog.asCatalog()); + adminService.createCatalog(createRequest); doTestInsufficientPrivileges( List.of( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 61a26b1c7b..d908752812 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -48,6 +48,7 @@ import org.apache.iceberg.types.Types; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; @@ -258,12 +259,14 @@ public void before(TestInfo testInfo) { .build(); catalogEntity = adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setCatalogType("INTERNAL") - .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setCatalogType("INTERNAL") + .setDefaultBaseLocation(storageLocation) + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build() + .asCatalog())); initBaseCatalog(); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java index ddcdf075d3..311c239902 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java @@ -45,6 +45,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizerImpl; @@ -205,16 +206,19 @@ public void before(TestInfo testInfo) { .build(); catalogEntity = adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), + "true") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 207955c932..b8742404b6 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -50,6 +50,7 @@ import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.iceberg.view.ImmutableSQLViewRepresentation; import org.apache.iceberg.view.ImmutableViewVersion; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; import org.apache.polaris.core.admin.model.StorageConfigInfo; @@ -1724,12 +1725,14 @@ public void testSendNotificationSufficientPrivileges() { .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) .build(); adminService.createCatalog( - new CatalogEntity.Builder() - .setName(externalCatalog) - .setDefaultBaseLocation(storageLocation) - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .setCatalogType("EXTERNAL") - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(externalCatalog) + .setDefaultBaseLocation(storageLocation) + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .setCatalogType("EXTERNAL") + .build() + .asCatalog())); adminService.createCatalogRole( externalCatalog, new CatalogRoleEntity.Builder().setName(CATALOG_ROLE1).build()); adminService.createCatalogRole( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 4a5506fa22..eedc9a588d 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -74,6 +74,7 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizerImpl; @@ -253,16 +254,19 @@ public void before(TestInfo testInfo) { .build(); catalogEntity = adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), + "true") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build() + .asCatalog())); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(createMockMetaStoreManagerFactory()); @@ -893,10 +897,12 @@ public void testUpdateNotificationCreateTableWithLocalFilePrefix() { String catalogWithoutStorage = "catalogWithoutStorage"; PolarisEntity catalogEntity = adminService.createCatalog( - new CatalogEntity.Builder() - .setDefaultBaseLocation("file://") - .setName(catalogWithoutStorage) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setDefaultBaseLocation("file://") + .setName(catalogWithoutStorage) + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( @@ -957,10 +963,12 @@ public void testUpdateNotificationCreateTableWithHttpPrefix() { String catalogName = "catalogForMaliciousDomain"; adminService.createCatalog( - new CatalogEntity.Builder() - .setDefaultBaseLocation("http://maliciousdomain.com") - .setName(catalogName) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setDefaultBaseLocation("http://maliciousdomain.com") + .setName(catalogName) + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( @@ -1496,16 +1504,19 @@ public void testDropTableWithPurgeDisabled() { .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .build(); adminService.createCatalog( - new CatalogEntity.Builder() - .setName(noPurgeCatalogName) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty(FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") - .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(noPurgeCatalogName) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .addProperty(FeatureConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") + .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, noPurgeCatalogName); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 7d817bec4f..fa51a021d8 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -40,6 +40,7 @@ import org.apache.iceberg.view.ViewCatalogTests; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; @@ -169,17 +170,20 @@ public void before(TestInfo testInfo) { securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .addProperty(FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setDefaultBaseLocation("file://tmp") - .setStorageConfigurationInfo( - new FileStorageConfigInfo( - StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://", "/", "*")), - "file://tmp") - .build()); + new CreateCatalogRequest( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .setDefaultBaseLocation("file://tmp") + .setStorageConfigurationInfo( + new FileStorageConfigInfo( + StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://", "/", "*")), + "file://tmp") + .build() + .asCatalog())); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index effcededd4..0ed13fc1aa 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -48,6 +48,7 @@ import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.CatalogGrant; import org.apache.polaris.core.admin.model.CatalogPrivilege; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.NamespacePrivilege; @@ -559,7 +560,9 @@ private boolean catalogOverlapsWithExistingCatalog(CatalogEntity catalogEntity) }); } - public PolarisEntity createCatalog(PolarisEntity entity) { + public PolarisEntity createCatalog(CreateCatalogRequest catalogRequest) { + PolarisEntity entity = CatalogEntity.fromCatalog(catalogRequest.getCatalog()); + PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_CATALOG; authorizeBasicRootOperationOrThrow(op); diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 8b92956f8e..b47d50208f 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -124,9 +124,7 @@ public Response createCatalog( PolarisAdminService adminService = newAdminService(realmContext, securityContext); Catalog catalog = request.getCatalog(); validateStorageConfig(catalog.getStorageConfigInfo()); - Catalog newCatalog = - new CatalogEntity(adminService.createCatalog(CatalogEntity.fromCatalog(catalog))) - .asCatalog(); + Catalog newCatalog = new CatalogEntity(adminService.createCatalog(request)).asCatalog(); LOGGER.info("Created new catalog {}", newCatalog); return Response.status(Response.Status.CREATED).build(); } From 4ad235e4a6223b2e51ec65993aaa517d46255a8c Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Sun, 30 Mar 2025 02:29:02 +0000 Subject: [PATCH 08/17] Add new interface UserSecretsManager along with a default implementation The default UnsafeInMemorySecretsManager just uses an inmemory ConcurrentHashMap to store secrets, but structurally illustrates the full flow of intended implementations. For mutual protection against a compromise of a secret store or the core persistence store, the default implementation demonstrates storing only an encrypted secret in the secret store, and a one-time-pad key in the returned referencePayload; other implementations using standard crypto protocols may choose to instead only utilize the remote secret store as the encryption keystore while storing the ciphertext in the referencePayload (e.g. using a KMS engine with Vault vs using a KV engine). Additionally, it demonstrates the use of an integrity check by storing a basic hashCode in the referencePayload as well. --- .../secrets/UnsafeInMemorySecretsManager.java | 152 ++++++++++++++++++ .../core/secrets/UserSecretReference.java | 94 +++++++++++ .../core/secrets/UserSecretsManager.java | 61 +++++++ .../UnsafeInMemorySecretsManagerTest.java | 26 +++ .../secrets/UserSecretsManagerBaseTest.java | 120 ++++++++++++++ 5 files changed, 453 insertions(+) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManagerTest.java create mode 100644 polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java new file mode 100644 index 0000000000..f0657c4f98 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import java.io.UnsupportedEncodingException; +import java.security.SecureRandom; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.polaris.core.entity.PolarisEntity; + +/** + * A minimal in-memory implementation of UserSecretsManager that should only be used for test and + * development purposes. + */ +public class UnsafeInMemorySecretsManager implements UserSecretsManager { + private final Map rawSecretStore = new ConcurrentHashMap<>(); + private final SecureRandom rand = new SecureRandom(); + + // Keys for information stored in referencePayload + private static final String CIPHERTEXT_HASH = "ciphertext-hash"; + private static final String ENCRYPTION_KEY = "encryption-key"; + + /** {@inheritDoc} */ + @Override + public UserSecretReference writeSecret(String secret, PolarisEntity forEntity) { + // For illustrative purposes and to exercise the control flow of requiring both the stored + // secret as well as the secretReferencePayload to recover the original secret, we'll use + // basic XOR encryption and store the randomly generated key in the reference payload. + // A production implementation will typically use a standard crypto library if applicable. + byte[] secretBytes; + try { + secretBytes = secret.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + byte[] oneTimeKey = new byte[secretBytes.length]; + byte[] cipherTextBytes = new byte[secretBytes.length]; + + // Generate one-time key of length equal to the original secret's bytes. + rand.nextBytes(oneTimeKey); + + // XOR all the bytes to generate the cipherText + for (int i = 0; i < cipherTextBytes.length; ++i) { + cipherTextBytes[i] = (byte) (secretBytes[i] ^ oneTimeKey[i]); + } + + // Store as Base64 since raw bytes won't play well with non-invertible String behaviors + // related to charset encodings. + String encryptedSecretCipherTextBase64 = Base64.getEncoder().encodeToString(cipherTextBytes); + String encryptedSecretKeyBase64 = Base64.getEncoder().encodeToString(oneTimeKey); + + String secretUrn; + for (int secretOrdinal = 0; ; ++secretOrdinal) { + secretUrn = + String.format( + "urn:polaris-secret:unsafe-in-memory:%d:%d", forEntity.getId(), secretOrdinal); + + // Store the base64-encoded encrypted ciphertext in the simulated "secret store". + String existingSecret = + rawSecretStore.putIfAbsent(secretUrn, encryptedSecretCipherTextBase64); + + // If there was already something stored under the current URN, continue to loop with + // an incremented ordinal suffix until we find an unused URN. + if (existingSecret == null) { + break; + } + } + + Map referencePayload = new HashMap<>(); + + // Keep a hash to detect data corruption or tampering; String::hashCode is standardized and can + // help detect systematic bugs causing corrupted secrets even if not secure against intentional + // tampering. + // A production implementation should use a cryptographic hash function instead if integrity + // of the secret is an actual concern. + referencePayload.put( + CIPHERTEXT_HASH, Integer.toString(encryptedSecretCipherTextBase64.hashCode())); + + // Keep the randomly generated one-time-use encryption key in the reference payload. + // A production implementation may choose to store an encryption key reference or URN if the + // key is ever shared and/or the key isn't a one-time-pad of the same length as the source + // secret. + referencePayload.put(ENCRYPTION_KEY, encryptedSecretKeyBase64); + UserSecretReference secretReference = new UserSecretReference(secretUrn, referencePayload); + return secretReference; + } + + /** {@inheritDoc} */ + @Override + public String readSecret(UserSecretReference secretReference) { + // TODO: Precondition checks and/or wire in PolarisDiagnostics + String encryptedSecretCipherTextBase64 = rawSecretStore.get(secretReference.getUrn()); + if (encryptedSecretCipherTextBase64 == null) { + // Secret at this URN no longer exists. + return null; + } + + String encryptedSecretKeyBase64 = secretReference.getReferencePayload().get(ENCRYPTION_KEY); + + // Validate integrity of the base64-encoded ciphertext which was retrieved from the secret + // store against the hash we stored in the referencePayload. + int expecteCipherTextBase64Hash = + Integer.parseInt(secretReference.getReferencePayload().get(CIPHERTEXT_HASH)); + if (encryptedSecretCipherTextBase64.hashCode() != expecteCipherTextBase64Hash) { + throw new IllegalArgumentException( + String.format( + "Ciphertext hash mismatch for URN %s; expected %d got %d", + secretReference.getUrn(), + expecteCipherTextBase64Hash, + encryptedSecretCipherTextBase64.hashCode())); + } + + byte[] cipherTextBytes = Base64.getDecoder().decode(encryptedSecretCipherTextBase64); + byte[] oneTimeKey = Base64.getDecoder().decode(encryptedSecretKeyBase64); + byte[] secretBytes = new byte[cipherTextBytes.length]; + + // XOR all the bytes to recover the secret + for (int i = 0; i < cipherTextBytes.length; ++i) { + secretBytes[i] = (byte) (cipherTextBytes[i] ^ oneTimeKey[i]); + } + + try { + return new String(secretBytes, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + /** {@inheritDoc} */ + @Override + public void deleteSecret(UserSecretReference secretReference) { + rawSecretStore.remove(secretReference.getUrn()); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java new file mode 100644 index 0000000000..f4faeb5062 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Represents a "wrapped reference" to a user-owned secret that holds an identifier to retrieve + * possibly remotely-stored secret material, along with an open-ended "referencePayload" that is + * specific to an implementation of the secret storage and which is needed "unwrap" the actual + * secret in combination with whatever is stored in the remote secrets storage. + * + *

Example scenarios: + * + *

If an implementation simply stores secrets directly in the secrets manager, the + * referencePayload may be empty and "unwrapping" would be a simple identity/no-op transformation. + * + *

If tampering or corruption of secrets in the secrets manager presents a unique threat, an + * implementation may use the referencePayload to ensure data integrity of the secret by storing a + * checksum or hash of the stored secret. + * + *

If the system must protect against independent exfiltration/attacks on a dedicated secrets + * manager and the core persistence database, the referencePayload may be used to coordinate + * secondary encryption keys such that the original secret can only be fully "unwrapped" given both + * the stored "secret material" as well as the referencePayload and any associated keys used for + * encryption. + */ +public class UserSecretReference { + @JsonProperty(value = "urn") + private final String urn; + + @JsonProperty(value = "referencePayload") + private final Map referencePayload; + + public UserSecretReference( + @JsonProperty(value = "urn", required = true) @Nonnull String urn, + @JsonProperty(value = "referencePayload") @Nullable Map referencePayload) { + this.urn = urn; + this.referencePayload = Objects.requireNonNullElse(referencePayload, new HashMap<>()); + } + + public @Nonnull String getUrn() { + return urn; + } + + public @Nonnull Map getReferencePayload() { + return referencePayload; + } + + @Override + public int hashCode() { + return Objects.hashCode(getUrn()) ^ Objects.hashCode(getReferencePayload()); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof UserSecretReference)) { + return false; + } + UserSecretReference that = (UserSecretReference) obj; + return Objects.equals(this.getUrn(), that.getUrn()) + && Objects.equals(this.getReferencePayload(), that.getReferencePayload()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("urn", getUrn()) + .add("referencePayload", String.format("", getReferencePayload().size())) + .toString(); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java new file mode 100644 index 0000000000..1dfd11d213 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import org.apache.polaris.core.entity.PolarisEntity; + +/** + * Manages secrets specified by users of the Polaris API, either directly or as an intermediary + * layer between Polaris and external secret-management systems. Such secrets are distinct from + * "service-level" secrets that pertain to the Polaris service itself which would be more statically + * configured system-wide. In contrast, user-owned secrets are handled dynamically as part of + * runtime API requests. + */ +public interface UserSecretsManager { + /** + * Persist the {@code secret} under a new URN {@code secretUrn} and return a {@code + * UserSecretReference} that can subsequently be used by this same UserSecretsManager to retrieve + * the original secret. The {@code forEntity} is provided for an implementation to optionally + * extract other identifying metadata such as entity type, name, etc., to store alongside the + * remotely stored secret to facilitate operational management of the secrets outside of the core + * Polaris service (for example, to perform garbage-collection if the Polaris service fails to + * delete managed secrets in the external system when associated entities are deleted. + * + * @param secret The secret to store + * @param forEntity The PolarisEntity that is associated with the secret + * @return A reference object that can be used to retrieve the secret which is safe to store in + * its entirety within a persisted PolarisEntity + */ + UserSecretReference writeSecret(String secret, PolarisEntity forEntity); + + /** + * Retrieve a secret using the {@code secretReference}. + * + * @param secretReference Identifier and any associated payload used for retrieving the secret + * @return The stored secret, or null if it no longer exists + */ + String readSecret(UserSecretReference secretReference); + + /** + * Delete a stored secret + * + * @param secretReference Identifier and any associated payload used for retrieving the secret + */ + void deleteSecret(UserSecretReference secretReference); +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManagerTest.java b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManagerTest.java new file mode 100644 index 0000000000..bac1fd6491 --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManagerTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +class UnsafeInMemorySecretsManagerTest extends UserSecretsManagerBaseTest { + @Override + protected UserSecretsManager newSecretsManager() { + return new UnsafeInMemorySecretsManager(); + } +} diff --git a/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java new file mode 100644 index 0000000000..2bba546ac7 --- /dev/null +++ b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Base test class for implementations of UserSecretsManager which can be extended by different + * implementation-specific unittests. + */ +public abstract class UserSecretsManagerBaseTest { + private static final ObjectMapper DEFAULT_MAPPER = new ObjectMapper(); + + /** + * @return a fresh instance of a UserSecretsManager to use in test cases. + */ + protected abstract UserSecretsManager newSecretsManager(); + + @Test + public void testBasicSecretStorageAndRetrieval() throws JsonProcessingException { + UserSecretsManager secretsManager = newSecretsManager(); + + PolarisEntity entity1 = new CatalogEntity.Builder() + .setId(1111L) + .setName("entity1") + .build(); + PolarisEntity entity2 = new CatalogEntity.Builder() + .setId(2222L) + .setName("entity2") + .build(); + + String secret1 = "sensitivesecret1"; + String secret2 = "sensitivesecret2"; + + UserSecretReference reference1 = secretsManager.writeSecret(secret1, entity1); + UserSecretReference reference2 = secretsManager.writeSecret(secret2, entity2); + + // Make sure we can JSON-serialize and deserialize the UserSecretReference objects. + String serializedReference1 = DEFAULT_MAPPER.writeValueAsString(reference1); + String serializedReference2 = DEFAULT_MAPPER.writeValueAsString(reference2); + + UserSecretReference reassembledReference1 = + DEFAULT_MAPPER.readValue(serializedReference1, UserSecretReference.class); + UserSecretReference reassembledReference2 = + DEFAULT_MAPPER.readValue(serializedReference2, UserSecretReference.class); + + Assertions.assertThat(reassembledReference1) + .isEqualTo(reference1); + Assertions.assertThat(reassembledReference2) + .isEqualTo(reference2); + Assertions.assertThat(secretsManager.readSecret(reassembledReference1)) + .isEqualTo(secret1); + Assertions.assertThat(secretsManager.readSecret(reassembledReference2)) + .isEqualTo(secret2); + } + + @Test + public void testMultipleSecretsForSameEntity() { + UserSecretsManager secretsManager = newSecretsManager(); + + PolarisEntity entity1 = new CatalogEntity.Builder() + .setId(1111L) + .setName("entity1") + .build(); + + String secret1 = "sensitivesecret1"; + String secret2 = "sensitivesecret2"; + + UserSecretReference reference1 = secretsManager.writeSecret(secret1, entity1); + UserSecretReference reference2 = secretsManager.writeSecret(secret2, entity1); + + Assertions.assertThat(secretsManager.readSecret(reference1)) + .isEqualTo(secret1); + Assertions.assertThat(secretsManager.readSecret(reference2)) + .isEqualTo(secret2); + } + + @Test + public void testDeleteSecret() { + UserSecretsManager secretsManager = newSecretsManager(); + + PolarisEntity entity1 = new CatalogEntity.Builder() + .setId(1111L) + .setName("entity1") + .build(); + + String secret1 = "sensitivesecret1"; + + UserSecretReference reference1 = secretsManager.writeSecret(secret1, entity1); + + Assertions.assertThat(secretsManager.readSecret(reference1)) + .isEqualTo(secret1); + + secretsManager.deleteSecret(reference1); + Assertions.assertThat(secretsManager.readSecret(reference1)) + .as("Deleted secret should return null") + .isNull(); + } +} From cc17ae579a53f877c5319a075404bcec9d82bc12 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Sun, 30 Mar 2025 07:52:17 +0000 Subject: [PATCH 09/17] Wire in UserSecretsManager to createCatalog and federated Iceberg API handlers Update the internal DPOs corresponding to the various ConnectionConfigInfo API objects to no longer contain any possible fields for inline secrets, instead holding the JSON-serializable UserSecretReference corresponding to external/offloaded secrets. CreateCatalog for federated catalogs containing secrets will now first extract UserSecretReferences from the CreateCatalogRequest, and the CatalogEntity will populate the DPOs corresponding to ConnectionConfigInfos in a secondary pass by pulling out the relevant extracted UserSecretReferences. For federated catalog requests, when reconstituting the actual sensitive secret configs, the UserSecretsManager will be used to obtain the secrets by using the stored UserSecretReferences. Remove vestigial internal properties from earlier prototypes. --- .../IcebergCatalogPropertiesProvider.java | 3 +- ...cebergRestConnectionConfigurationInfo.java | 6 +- .../PolarisAuthenticationParameters.java | 14 +- ...PolarisBearerAuthenticationParameters.java | 24 ++-- ...larisOAuthClientCredentialsParameters.java | 33 ++--- .../polaris/core/entity/CatalogEntity.java | 46 +------ .../TransactionalMetaStoreManagerImpl.java | 18 --- .../secrets/UnsafeInMemorySecretsManager.java | 4 + ...olarisConnectionConfigurationInfoTest.java | 20 ++- .../secrets/UserSecretsManagerBaseTest.java | 41 ++---- .../service/admin/PolarisAdminService.java | 122 +++++++++++++++++- .../iceberg/IcebergCatalogHandler.java | 15 ++- 12 files changed, 214 insertions(+), 132 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java index 74953ae560..df6897854e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java @@ -20,8 +20,9 @@ import jakarta.annotation.Nonnull; import java.util.Map; +import org.apache.polaris.core.secrets.UserSecretsManager; public interface IcebergCatalogPropertiesProvider { @Nonnull - Map asIcebergCatalogProperties(); + Map asIcebergCatalogProperties(UserSecretsManager secretsManager); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java index 35609c5fb7..b7c8609333 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java @@ -27,6 +27,7 @@ import org.apache.iceberg.CatalogProperties; import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; +import org.apache.polaris.core.secrets.UserSecretsManager; public class IcebergRestConnectionConfigurationInfo extends PolarisConnectionConfigurationInfo implements IcebergCatalogPropertiesProvider { @@ -57,13 +58,14 @@ public PolarisAuthenticationParameters getAuthenticationParameters() { } @Override - public @Nonnull Map asIcebergCatalogProperties() { + public @Nonnull Map asIcebergCatalogProperties( + UserSecretsManager secretsManager) { HashMap properties = new HashMap<>(); properties.put(CatalogProperties.URI, getUri()); if (getRemoteCatalogName() != null) { properties.put(CatalogProperties.WAREHOUSE_LOCATION, getRemoteCatalogName()); } - properties.putAll(authenticationParameters.asIcebergCatalogProperties()); + properties.putAll(authenticationParameters.asIcebergCatalogProperties(secretsManager)); return properties; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java index adf49edd81..6bdb788adf 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java @@ -22,9 +22,11 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import jakarta.annotation.Nonnull; +import java.util.Map; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; +import org.apache.polaris.core.secrets.UserSecretReference; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "authenticationType", visible = true) @JsonSubTypes({ @@ -33,6 +35,9 @@ }) public abstract class PolarisAuthenticationParameters implements IcebergCatalogPropertiesProvider { + public static final String INLINE_CLIENT_SECRET_REFERENCE_KEY = "inlineClientSecretReference"; + public static final String INLINE_BEARER_TOKEN_REFERENCE_KEY = "inlineBearerTokenReference"; + @JsonProperty(value = "authenticationType") private final AuthenticationType authenticationType; @@ -48,8 +53,9 @@ public PolarisAuthenticationParameters( public abstract AuthenticationParameters asAuthenticationParametersModel(); - public static PolarisAuthenticationParameters fromAuthenticationParametersModel( - AuthenticationParameters authenticationParameters) { + public static PolarisAuthenticationParameters fromAuthenticationParametersModelWithSecrets( + AuthenticationParameters authenticationParameters, + Map secretReferences) { PolarisAuthenticationParameters config = null; switch (authenticationParameters.getAuthenticationType()) { case OAUTH: @@ -60,7 +66,7 @@ public static PolarisAuthenticationParameters fromAuthenticationParametersModel( AuthenticationType.OAUTH, oauthClientCredentialsModel.getTokenUri(), oauthClientCredentialsModel.getClientId(), - oauthClientCredentialsModel.getClientSecret(), + secretReferences.get(INLINE_CLIENT_SECRET_REFERENCE_KEY), oauthClientCredentialsModel.getScopes()); break; case BEARER: @@ -68,7 +74,7 @@ public static PolarisAuthenticationParameters fromAuthenticationParametersModel( (BearerAuthenticationParameters) authenticationParameters; config = new PolarisBearerAuthenticationParameters( - AuthenticationType.BEARER, bearerAuthenticationParametersModel.getBearerToken()); + AuthenticationType.BEARER, secretReferences.get(INLINE_BEARER_TOKEN_REFERENCE_KEY)); break; default: throw new IllegalStateException( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java index 2c4dbea7ca..c3d9ec3002 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java @@ -25,35 +25,38 @@ import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; +import org.apache.polaris.core.secrets.UserSecretReference; +import org.apache.polaris.core.secrets.UserSecretsManager; public class PolarisBearerAuthenticationParameters extends PolarisAuthenticationParameters { - @JsonProperty(value = "bearerToken") - private final String bearerToken; + @JsonProperty(value = "bearerTokenReference") + private final UserSecretReference bearerTokenReference; public PolarisBearerAuthenticationParameters( @JsonProperty(value = "authenticationType", required = true) @Nonnull AuthenticationType authenticationType, - @JsonProperty(value = "bearerToken", required = true) @Nonnull String bearerToken) { + @JsonProperty(value = "bearerTokenReference", required = true) @Nonnull + UserSecretReference bearerTokenReference) { super(authenticationType); - this.bearerToken = bearerToken; + this.bearerTokenReference = bearerTokenReference; } - public @Nonnull String getBearerToken() { - return bearerToken; + public @Nonnull UserSecretReference getBearerTokenReference() { + return bearerTokenReference; } @Override - public @Nonnull Map asIcebergCatalogProperties() { - return Map.of(OAuth2Properties.TOKEN, getBearerToken()); + public @Nonnull Map asIcebergCatalogProperties( + UserSecretsManager secretsManager) { + String bearerToken = secretsManager.readSecret(getBearerTokenReference()); + return Map.of(OAuth2Properties.TOKEN, bearerToken); } @Override public AuthenticationParameters asAuthenticationParametersModel() { - // TODO: redact secrets from the model return BearerAuthenticationParameters.builder() .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.BEARER) - .setBearerToken(getBearerToken()) .build(); } @@ -61,6 +64,7 @@ public AuthenticationParameters asAuthenticationParametersModel() { public String toString() { return MoreObjects.toStringHelper(this) .add("authenticationType", getAuthenticationType()) + .add("bearerTokenReference", getBearerTokenReference()) .toString(); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java index 664266d4a3..c7f1bc593d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java @@ -35,6 +35,8 @@ import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; +import org.apache.polaris.core.secrets.UserSecretReference; +import org.apache.polaris.core.secrets.UserSecretsManager; public class PolarisOAuthClientCredentialsParameters extends PolarisAuthenticationParameters { @@ -46,8 +48,8 @@ public class PolarisOAuthClientCredentialsParameters extends PolarisAuthenticati @JsonProperty(value = "clientId") private final String clientId; - @JsonProperty(value = "clientSecret") - private final String clientSecret; + @JsonProperty(value = "clientSecretReference") + private final UserSecretReference clientSecretReference; @JsonProperty(value = "scopes") private final List scopes; @@ -57,13 +59,14 @@ public PolarisOAuthClientCredentialsParameters( AuthenticationType authenticationType, @JsonProperty(value = "tokenUri", required = false) @Nullable String tokenUri, @JsonProperty(value = "clientId", required = true) @Nonnull String clientId, - @JsonProperty(value = "clientSecret", required = true) @Nonnull String clientSecret, + @JsonProperty(value = "clientSecretReference", required = true) @Nonnull + UserSecretReference clientSecretReference, @JsonProperty(value = "scopes", required = false) @Nullable List scopes) { super(authenticationType); this.tokenUri = tokenUri; this.clientId = clientId; - this.clientSecret = clientSecret; + this.clientSecretReference = clientSecretReference; this.scopes = scopes; validateTokenUri(tokenUri); @@ -77,39 +80,39 @@ public PolarisOAuthClientCredentialsParameters( return clientId; } - public @Nonnull String getClientSecret() { - return clientSecret; + public @Nonnull UserSecretReference getClientSecretReference() { + return clientSecretReference; } public @Nonnull List getScopes() { return scopes; } - @JsonIgnore - public @Nonnull String getCredential() { - return COLON_JOINER.join(clientId, clientSecret); - } - @JsonIgnore public @Nonnull String getScopesAsString() { return OAuth2Util.toScope( Objects.requireNonNullElse(scopes, List.of(OAuth2Properties.CATALOG_SCOPE))); } + private @Nonnull String getCredentialAsConcatenatedString(UserSecretsManager secretsManager) { + String clientSecret = secretsManager.readSecret(getClientSecretReference()); + return COLON_JOINER.join(clientId, clientSecret); + } + @Override - public @Nonnull Map asIcebergCatalogProperties() { + public @Nonnull Map asIcebergCatalogProperties( + UserSecretsManager secretsManager) { HashMap properties = new HashMap<>(); if (getTokenUri() != null) { properties.put(OAuth2Properties.OAUTH2_SERVER_URI, getTokenUri()); } - properties.put(OAuth2Properties.CREDENTIAL, getCredential()); + properties.put(OAuth2Properties.CREDENTIAL, getCredentialAsConcatenatedString(secretsManager)); properties.put(OAuth2Properties.SCOPE, getScopesAsString()); return properties; } @Override public AuthenticationParameters asAuthenticationParametersModel() { - // TODO: redact secrets from the model return OAuthClientCredentialsParameters.builder() .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.OAUTH) .setTokenUri(getTokenUri()) @@ -137,7 +140,7 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("tokenUri", getTokenUri()) .add("clientId", getClientId()) - .add("clientSecret", getClientSecret()) + .add("clientSecretReference", getClientSecretReference()) .add("scopes", getScopesAsString()) .toString(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index 3dabbfb8e0..5ee19baf13 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -47,6 +47,7 @@ import org.apache.polaris.core.connection.IcebergRestConnectionConfigurationInfo; import org.apache.polaris.core.connection.PolarisAuthenticationParameters; import org.apache.polaris.core.connection.PolarisConnectionConfigurationInfo; +import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.storage.FileStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; @@ -72,13 +73,6 @@ public class CatalogEntity extends PolarisEntity { public static final String REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY = "replace-new-location-prefix-with-catalog-default"; - // TODO: Refactor all these into ConnectionConfigurationInfo - public static final String CONNECTION_REMOTE_URI_KEY = "connection.uri"; - public static final String CONNECTION_CLIENT_ID_KEY = "connection.clientId"; - public static final String CONNECTION_CLIENT_SECRET_KEY = "connection.clientSecret"; - public static final String CONNECTION_CATALOG_NAME_KEY = "connection.catalogName"; - public static final String CONNECTION_SCOPES_KEY = "connection.scopes"; - public CatalogEntity(PolarisBaseEntity sourceEntity) { super(sourceEntity); } @@ -91,7 +85,6 @@ public static CatalogEntity of(PolarisBaseEntity sourceEntity) { } public static CatalogEntity fromCatalog(Catalog catalog) { - Builder builder = new Builder() .setName(catalog.getName()) @@ -102,9 +95,6 @@ public static CatalogEntity fromCatalog(Catalog catalog) { builder.setInternalProperties(internalProperties); builder.setStorageConfigurationInfo( catalog.getStorageConfigInfo(), getDefaultBaseLocation(catalog)); - if (catalog instanceof ExternalCatalog) { - builder.setConnectionConfigurationInfo(((ExternalCatalog) catalog).getConnectionConfigInfo()); - } return builder.build(); } @@ -232,31 +222,6 @@ public PolarisConnectionConfigurationInfo getConnectionConfigurationInfo() { return null; } - public String getConnectionUri() { - // TODO: Refactor this to use new ConnectionConfigurationInfo - return getPropertiesAsMap().get(CONNECTION_REMOTE_URI_KEY); - } - - public String getConnectionClientId() { - // TODO: Refactor this to use new ConnectionConfigurationInfo - return getInternalPropertiesAsMap().get(CONNECTION_CLIENT_ID_KEY); - } - - public String getConnectionClientSecret() { - // TODO: Refactor this to use new ConnectionConfigurationInfo - return getInternalPropertiesAsMap().get(CONNECTION_CLIENT_SECRET_KEY); - } - - public String getConnectionCatalogName() { - // TODO: Refactor this to use new ConnectionConfigurationInfo - return getPropertiesAsMap().get(CONNECTION_CATALOG_NAME_KEY); - } - - public String getConnectionScopes() { - // TODO: Refactor this to use new ConnectionConfigurationInfo - return getPropertiesAsMap().get(CONNECTION_SCOPES_KEY); - } - public static class Builder extends PolarisEntity.BaseBuilder { public Builder() { super(); @@ -357,8 +322,9 @@ private void validateMaxAllowedLocations(Collection allowedLocations) { } } - public Builder setConnectionConfigurationInfo( - ConnectionConfigInfo connectionConfigurationModel) { + public Builder setConnectionConfigurationInfoWithSecrets( + ConnectionConfigInfo connectionConfigurationModel, + Map secretReferences) { if (connectionConfigurationModel != null) { PolarisConnectionConfigurationInfo config; switch (connectionConfigurationModel.getConnectionType()) { @@ -366,8 +332,8 @@ public Builder setConnectionConfigurationInfo( IcebergRestConnectionConfigInfo icebergRestConfigModel = (IcebergRestConnectionConfigInfo) connectionConfigurationModel; PolarisAuthenticationParameters authenticationParameters = - PolarisAuthenticationParameters.fromAuthenticationParametersModel( - icebergRestConfigModel.getAuthenticationParameters()); + PolarisAuthenticationParameters.fromAuthenticationParametersModelWithSecrets( + icebergRestConfigModel.getAuthenticationParameters(), secretReferences); config = new IcebergRestConnectionConfigurationInfo( ConnectionType.ICEBERG_REST, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index b777a204b9..27b049e917 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -33,7 +33,6 @@ import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; -import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; @@ -447,23 +446,6 @@ private void revokeGrantRecord( ms.persistStorageIntegrationIfNeededInCurrentTxn(callCtx, catalog, integration); - // TODO: Push this into an IntegrationPersistence flow which prepares an external secret store - // and swaps it with a reference in internalProperties which can be used to reconstitue the - // secret. - if (catalog.getPropertiesAsMap().containsKey(CatalogEntity.CONNECTION_CLIENT_SECRET_KEY)) { - catalog = - new PolarisEntity.Builder(PolarisEntity.of(catalog)) - .addInternalProperty( - CatalogEntity.CONNECTION_CLIENT_ID_KEY, - catalog.getPropertiesAsMap().get(CatalogEntity.CONNECTION_CLIENT_ID_KEY)) - .addInternalProperty( - CatalogEntity.CONNECTION_CLIENT_SECRET_KEY, - catalog.getPropertiesAsMap().get(CatalogEntity.CONNECTION_CLIENT_SECRET_KEY)) - .addProperty(CatalogEntity.CONNECTION_CLIENT_ID_KEY, "") - .addProperty(CatalogEntity.CONNECTION_CLIENT_SECRET_KEY, "") - .build(); - } - // now create and persist new catalog entity this.persistNewEntity(callCtx, ms, catalog); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java index f0657c4f98..cc8a6b299a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java @@ -31,6 +31,10 @@ * development purposes. */ public class UnsafeInMemorySecretsManager implements UserSecretsManager { + // TODO: Remove this and wire into QuarkusProducers; just a placeholder for now to get the + // rest of the logic working. + public static final UserSecretsManager GLOBAL_INSTANCE = new UnsafeInMemorySecretsManager(); + private final Map rawSecretStore = new ConcurrentHashMap<>(); private final SecureRandom rand = new SecureRandom(); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java index d9e0febebe..03c5220cb4 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java @@ -21,13 +21,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; public class PolarisConnectionConfigurationInfoTest { - PolarisDiagnostics polarisDiagnostics = Mockito.mock(PolarisDiagnostics.class); + PolarisDiagnostics polarisDiagnostics = new PolarisDefaultDiagServiceImpl(); ObjectMapper objectMapper = new ObjectMapper(); @Test @@ -42,7 +42,13 @@ void testOAuthClientCredentialsParameters() throws JsonProcessingException { + " \"authenticationType\": \"OAUTH\"," + " \"tokenUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog/v1/oauth/tokens\"," + " \"clientId\": \"client-id\"," - + " \"clientSecret\": \"client-secret\"," + + " \"clientSecretReference\": {" + + " \"urn\": \"urn:polaris-secret:keystore-id-12345\"," + + " \"referencePayload\": {" + + " \"hash\": \"a1b2c3\"," + + " \"encryption-key\": \"z0y9x8\"" + + " }" + + " }," + " \"scopes\": [\"PRINCIPAL_ROLE:ALL\"]" + " }" + "}"; @@ -65,7 +71,13 @@ void testBearerAuthenticationParameters() throws JsonProcessingException { + " \"remoteCatalogName\": \"my-catalog\"," + " \"authenticationParameters\": {" + " \"authenticationType\": \"BEARER\"," - + " \"bearerToken\": \"bearer-token\"" + + " \"bearerTokenReference\": {" + + " \"urn\": \"urn:polaris-secret:keystore-id-12345\"," + + " \"referencePayload\": {" + + " \"hash\": \"a1b2c3\"," + + " \"encryption-key\": \"z0y9x8\"" + + " }" + + " }" + " }" + "}"; PolarisConnectionConfigurationInfo connectionConfigurationInfo = diff --git a/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java index 2bba546ac7..09e45b185a 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/secrets/UserSecretsManagerBaseTest.java @@ -41,14 +41,8 @@ public abstract class UserSecretsManagerBaseTest { public void testBasicSecretStorageAndRetrieval() throws JsonProcessingException { UserSecretsManager secretsManager = newSecretsManager(); - PolarisEntity entity1 = new CatalogEntity.Builder() - .setId(1111L) - .setName("entity1") - .build(); - PolarisEntity entity2 = new CatalogEntity.Builder() - .setId(2222L) - .setName("entity2") - .build(); + PolarisEntity entity1 = new CatalogEntity.Builder().setId(1111L).setName("entity1").build(); + PolarisEntity entity2 = new CatalogEntity.Builder().setId(2222L).setName("entity2").build(); String secret1 = "sensitivesecret1"; String secret2 = "sensitivesecret2"; @@ -65,24 +59,17 @@ public void testBasicSecretStorageAndRetrieval() throws JsonProcessingException UserSecretReference reassembledReference2 = DEFAULT_MAPPER.readValue(serializedReference2, UserSecretReference.class); - Assertions.assertThat(reassembledReference1) - .isEqualTo(reference1); - Assertions.assertThat(reassembledReference2) - .isEqualTo(reference2); - Assertions.assertThat(secretsManager.readSecret(reassembledReference1)) - .isEqualTo(secret1); - Assertions.assertThat(secretsManager.readSecret(reassembledReference2)) - .isEqualTo(secret2); + Assertions.assertThat(reassembledReference1).isEqualTo(reference1); + Assertions.assertThat(reassembledReference2).isEqualTo(reference2); + Assertions.assertThat(secretsManager.readSecret(reassembledReference1)).isEqualTo(secret1); + Assertions.assertThat(secretsManager.readSecret(reassembledReference2)).isEqualTo(secret2); } @Test public void testMultipleSecretsForSameEntity() { UserSecretsManager secretsManager = newSecretsManager(); - PolarisEntity entity1 = new CatalogEntity.Builder() - .setId(1111L) - .setName("entity1") - .build(); + PolarisEntity entity1 = new CatalogEntity.Builder().setId(1111L).setName("entity1").build(); String secret1 = "sensitivesecret1"; String secret2 = "sensitivesecret2"; @@ -90,27 +77,21 @@ public void testMultipleSecretsForSameEntity() { UserSecretReference reference1 = secretsManager.writeSecret(secret1, entity1); UserSecretReference reference2 = secretsManager.writeSecret(secret2, entity1); - Assertions.assertThat(secretsManager.readSecret(reference1)) - .isEqualTo(secret1); - Assertions.assertThat(secretsManager.readSecret(reference2)) - .isEqualTo(secret2); + Assertions.assertThat(secretsManager.readSecret(reference1)).isEqualTo(secret1); + Assertions.assertThat(secretsManager.readSecret(reference2)).isEqualTo(secret2); } @Test public void testDeleteSecret() { UserSecretsManager secretsManager = newSecretsManager(); - PolarisEntity entity1 = new CatalogEntity.Builder() - .setId(1111L) - .setName("entity1") - .build(); + PolarisEntity entity1 = new CatalogEntity.Builder().setId(1111L).setName("entity1").build(); String secret1 = "sensitivesecret1"; UserSecretReference reference1 = secretsManager.writeSecret(secret1, entity1); - Assertions.assertThat(secretsManager.readSecret(reference1)) - .isEqualTo(secret1); + Assertions.assertThat(secretsManager.readSecret(reference1)).isEqualTo(secret1); secretsManager.deleteSecret(reference1); Assertions.assertThat(secretsManager.readSecret(reference1)) diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 0ed13fc1aa..28189d2d08 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -26,6 +26,7 @@ import jakarta.ws.rs.core.SecurityContext; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -46,12 +47,18 @@ import org.apache.iceberg.exceptions.ValidationException; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; +import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogGrant; import org.apache.polaris.core.admin.model.CatalogPrivilege; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.NamespacePrivilege; +import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; import org.apache.polaris.core.admin.model.TableGrant; @@ -67,6 +74,7 @@ import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.connection.PolarisAuthenticationParameters; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.CatalogRoleEntity; @@ -92,6 +100,9 @@ import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.secrets.UnsafeInMemorySecretsManager; +import org.apache.polaris.core.secrets.UserSecretReference; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; @@ -149,6 +160,11 @@ private PolarisCallContext getCurrentPolarisContext() { return callContext.getPolarisCallContext(); } + private UserSecretsManager getUserSecretsManager() { + // TODO: Wire into appropriate factories and/or contexts. + return UnsafeInMemorySecretsManager.GLOBAL_INSTANCE; + } + private Optional findCatalogByName(String name) { return Optional.ofNullable(resolutionManifest.getResolvedReferenceCatalogEntity()) .map(path -> CatalogEntity.of(path.getRawLeafEntity())); @@ -560,12 +576,89 @@ private boolean catalogOverlapsWithExistingCatalog(CatalogEntity catalogEntity) }); } - public PolarisEntity createCatalog(CreateCatalogRequest catalogRequest) { - PolarisEntity entity = CatalogEntity.fromCatalog(catalogRequest.getCatalog()); + /** + * Secrets embedded *or* simply referenced through the API model will require separate processing + * for normalizing into resolved/verified/offloaded UserSecretReference objects which are then + * placed appropriately into persistence objects. + * + *

If secrets are already direct URIs/URNs to an external secret store, we may need to validate + * the URI/URN and/or transform into a polaris-internal URN format along with type-information or + * other secrets-manager metadata in the referencePayload. + * + *

If secrets reference first-class Polaris-stored secrets, we must resolve the associated + * polaris persistence entities defining access to those secrets and perform authorization. + * + *

If secrets are provided inline as part of the request, we must explicitly offload the + * secrets into a Polaris service-level secrets manager and return the associated internal + * references to the stored secret. + */ + private Map extractSecretReferences( + CreateCatalogRequest catalogRequest, PolarisEntity forEntity) { + Map secretReferences = new HashMap<>(); + Catalog catalog = catalogRequest.getCatalog(); + UserSecretsManager secretsManager = getUserSecretsManager(); + if (catalog instanceof ExternalCatalog externalCatalog) { + if (externalCatalog.getConnectionConfigInfo() != null) { + ConnectionConfigInfo connectionConfig = externalCatalog.getConnectionConfigInfo(); + AuthenticationParameters authenticationParameters = + connectionConfig.getAuthenticationParameters(); + + switch (authenticationParameters.getAuthenticationType()) { + case OAUTH: + { + OAuthClientCredentialsParameters oauthClientCredentialsModel = + (OAuthClientCredentialsParameters) authenticationParameters; + String inlineClientSecret = oauthClientCredentialsModel.getClientSecret(); + UserSecretReference secretReference = + secretsManager.writeSecret(inlineClientSecret, forEntity); + secretReferences.put( + PolarisAuthenticationParameters.INLINE_CLIENT_SECRET_REFERENCE_KEY, + secretReference); + break; + } + case BEARER: + { + BearerAuthenticationParameters bearerAuthenticationParametersModel = + (BearerAuthenticationParameters) authenticationParameters; + String inlineBearerToken = bearerAuthenticationParametersModel.getBearerToken(); + UserSecretReference secretReference = + secretsManager.writeSecret(inlineBearerToken, forEntity); + secretReferences.put( + PolarisAuthenticationParameters.INLINE_BEARER_TOKEN_REFERENCE_KEY, + secretReference); + break; + } + default: + throw new IllegalStateException( + "Unsupported authentication type: " + + authenticationParameters.getAuthenticationType()); + } + } + } + return secretReferences; + } + /** + * @see #extractSecretReferences + */ + private boolean requiresSecretReferenceExtraction(CreateCatalogRequest catalogRequest) { + Catalog catalog = catalogRequest.getCatalog(); + if (catalog instanceof ExternalCatalog externalCatalog) { + if (externalCatalog.getConnectionConfigInfo() != null) { + // TODO: Make this more targeted once we have connection configs that don't involve + // processing of inline secrets. + return true; + } + } + return false; + } + + public PolarisEntity createCatalog(CreateCatalogRequest catalogRequest) { PolarisAuthorizableOperation op = PolarisAuthorizableOperation.CREATE_CATALOG; authorizeBasicRootOperationOrThrow(op); + CatalogEntity entity = CatalogEntity.fromCatalog(catalogRequest.getCatalog()); + checkArgument(entity.getId() == -1, "Entity to be created must have no ID assigned"); if (catalogOverlapsWithExistingCatalog((CatalogEntity) entity)) { @@ -574,14 +667,33 @@ public PolarisEntity createCatalog(CreateCatalogRequest catalogRequest) { entity.getName()); } - PolarisEntity polarisEntity = - new PolarisEntity.Builder(entity) + // After basic validations, now populate id and creation timestamp. + entity = + new CatalogEntity.Builder(entity) .setId(metaStoreManager.generateNewEntityId(getCurrentPolarisContext()).getId()) .setCreateTimestamp(System.currentTimeMillis()) .build(); + + if (requiresSecretReferenceExtraction(catalogRequest)) { + // TODO: Also gate this on a feature parameter + // For fields that contain references to secrets, we'll separately process the secrets from + // the original request first, and then populate those fields with the extracted secret + // references as part of the construction of the internal persistence entity. + Map processedSecretReferences = + extractSecretReferences(catalogRequest, entity); + entity = + new CatalogEntity.Builder(entity) + .setConnectionConfigurationInfoWithSecrets( + ((ExternalCatalog) catalogRequest.getCatalog()).getConnectionConfigInfo(), + processedSecretReferences) + .build(); + } + CreateCatalogResult catalogResult = - metaStoreManager.createCatalog(getCurrentPolarisContext(), polarisEntity, List.of()); + metaStoreManager.createCatalog(getCurrentPolarisContext(), entity, List.of()); if (catalogResult.alreadyExists()) { + // TODO: Proactive garbage-collection of any inline secrets that were written to the + // secrets manager, here and on any other unexpected exception as well. throw new AlreadyExistsException( "Cannot create Catalog %s. Catalog already exists or resolution failed", entity.getName()); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 673f1396fc..0ab9f64b17 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -74,6 +74,7 @@ import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.connection.IcebergRestConnectionConfigurationInfo; import org.apache.polaris.core.connection.PolarisConnectionConfigurationInfo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; @@ -85,6 +86,8 @@ import org.apache.polaris.core.persistence.TransactionWorkspaceMetaStoreManager; import org.apache.polaris.core.persistence.dao.entity.EntitiesResult; import org.apache.polaris.core.persistence.dao.entity.EntityWithPath; +import org.apache.polaris.core.secrets.UnsafeInMemorySecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.service.catalog.SupportsNotifications; import org.apache.polaris.service.catalog.common.CatalogHandler; @@ -156,6 +159,11 @@ public static boolean isCreate(UpdateTableRequest request) { return isCreate; } + private UserSecretsManager getUserSecretsManager() { + // TODO: Wire into appropriate factories and/or contexts. + return UnsafeInMemorySecretsManager.GLOBAL_INSTANCE; + } + @Override protected void initializeCatalog() { CatalogEntity resolvedCatalogEntity = @@ -179,14 +187,15 @@ protected void initializeCatalog() { HTTPClient.builder(config) .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) .build()); + federatedCatalog.initialize( + ((IcebergRestConnectionConfigurationInfo) connectionConfigurationInfo) + .getRemoteCatalogName(), + connectionConfigurationInfo.asIcebergCatalogProperties(getUserSecretsManager())); break; default: throw new UnsupportedOperationException( "Connection type not supported: " + connectionConfigurationInfo.getConnectionType()); } - federatedCatalog.initialize( - resolvedCatalogEntity.getConnectionCatalogName(), - connectionConfigurationInfo.asIcebergCatalogProperties()); this.baseCatalog = federatedCatalog; } else { LOGGER.atInfo().log("Initializing non-federated catalog"); From 6700fb293b3fc33d803f7b844b9623b5d237b440 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Thu, 3 Apr 2025 05:41:47 +0000 Subject: [PATCH 10/17] Since we already use commons-codec DigestUtils.sha256Hex, use that for the hash in UnsafeInMemorySecretsManager just for consistency and to illustrate a typical scenario using a cryptographic hash. --- .../secrets/UnsafeInMemorySecretsManager.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java index cc8a6b299a..d5cdfa0c3f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java @@ -23,7 +23,9 @@ import java.util.Base64; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.polaris.core.entity.PolarisEntity; /** @@ -90,13 +92,9 @@ public UserSecretReference writeSecret(String secret, PolarisEntity forEntity) { Map referencePayload = new HashMap<>(); - // Keep a hash to detect data corruption or tampering; String::hashCode is standardized and can - // help detect systematic bugs causing corrupted secrets even if not secure against intentional - // tampering. - // A production implementation should use a cryptographic hash function instead if integrity - // of the secret is an actual concern. - referencePayload.put( - CIPHERTEXT_HASH, Integer.toString(encryptedSecretCipherTextBase64.hashCode())); + // Keep a hash to detect data corruption or tampering; hash the base64-encoded string so + // we detect the corruption even before attempting to base64-decode it. + referencePayload.put(CIPHERTEXT_HASH, DigestUtils.sha256Hex(encryptedSecretCipherTextBase64)); // Keep the randomly generated one-time-use encryption key in the reference payload. // A production implementation may choose to store an encryption key reference or URN if the @@ -121,15 +119,15 @@ public String readSecret(UserSecretReference secretReference) { // Validate integrity of the base64-encoded ciphertext which was retrieved from the secret // store against the hash we stored in the referencePayload. - int expecteCipherTextBase64Hash = - Integer.parseInt(secretReference.getReferencePayload().get(CIPHERTEXT_HASH)); - if (encryptedSecretCipherTextBase64.hashCode() != expecteCipherTextBase64Hash) { + String expecteCipherTextBase64Hash = secretReference.getReferencePayload().get(CIPHERTEXT_HASH); + String retrievedCipherTextBase64Hash = DigestUtils.sha256Hex(encryptedSecretCipherTextBase64); + if (!Objects.equals(retrievedCipherTextBase64Hash, expecteCipherTextBase64Hash)) { throw new IllegalArgumentException( String.format( - "Ciphertext hash mismatch for URN %s; expected %d got %d", + "Ciphertext hash mismatch for URN %s; expected %s got %s", secretReference.getUrn(), expecteCipherTextBase64Hash, - encryptedSecretCipherTextBase64.hashCode())); + retrievedCipherTextBase64Hash)); } byte[] cipherTextBytes = Base64.getDecoder().decode(encryptedSecretCipherTextBase64); From 10b2966006b851d497b3dd07a883d146d7a988d7 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Fri, 4 Apr 2025 03:04:46 +0000 Subject: [PATCH 11/17] Rename the persistence-objects corresponding to API model objects with a new naming convention that just takes the API model object name and appends "Dpo" as a suffix; --- ....java => AuthenticationParametersDpo.java} | 16 +++++++------- ...=> BearerAuthenticationParametersDpo.java} | 4 ++-- ...Info.java => ConnectionConfigInfoDpo.java} | 16 +++++++------- ...> IcebergRestConnectionConfigInfoDpo.java} | 10 ++++----- ... OAuthClientCredentialsParametersDpo.java} | 4 ++-- .../polaris/core/entity/CatalogEntity.java | 21 +++++++++---------- ....java => ConnectionConfigInfoDpoTest.java} | 10 ++++----- .../service/admin/PolarisAdminService.java | 8 +++---- .../iceberg/IcebergCatalogHandler.java | 8 +++---- 9 files changed, 46 insertions(+), 51 deletions(-) rename polaris-core/src/main/java/org/apache/polaris/core/connection/{PolarisAuthenticationParameters.java => AuthenticationParametersDpo.java} (84%) rename polaris-core/src/main/java/org/apache/polaris/core/connection/{PolarisBearerAuthenticationParameters.java => BearerAuthenticationParametersDpo.java} (94%) rename polaris-core/src/main/java/org/apache/polaris/core/connection/{PolarisConnectionConfigurationInfo.java => ConnectionConfigInfoDpo.java} (85%) rename polaris-core/src/main/java/org/apache/polaris/core/connection/{IcebergRestConnectionConfigurationInfo.java => IcebergRestConnectionConfigInfoDpo.java} (90%) rename polaris-core/src/main/java/org/apache/polaris/core/connection/{PolarisOAuthClientCredentialsParameters.java => OAuthClientCredentialsParametersDpo.java} (97%) rename polaris-core/src/test/java/org/apache/polaris/core/connection/{PolarisConnectionConfigurationInfoTest.java => ConnectionConfigInfoDpoTest.java} (91%) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java similarity index 84% rename from polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java rename to polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java index 6bdb788adf..b9e45ae9ac 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisAuthenticationParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java @@ -30,10 +30,10 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "authenticationType", visible = true) @JsonSubTypes({ - @JsonSubTypes.Type(value = PolarisOAuthClientCredentialsParameters.class, name = "OAUTH"), - @JsonSubTypes.Type(value = PolarisBearerAuthenticationParameters.class, name = "BEARER"), + @JsonSubTypes.Type(value = OAuthClientCredentialsParametersDpo.class, name = "OAUTH"), + @JsonSubTypes.Type(value = BearerAuthenticationParametersDpo.class, name = "BEARER"), }) -public abstract class PolarisAuthenticationParameters implements IcebergCatalogPropertiesProvider { +public abstract class AuthenticationParametersDpo implements IcebergCatalogPropertiesProvider { public static final String INLINE_CLIENT_SECRET_REFERENCE_KEY = "inlineClientSecretReference"; public static final String INLINE_BEARER_TOKEN_REFERENCE_KEY = "inlineBearerTokenReference"; @@ -41,7 +41,7 @@ public abstract class PolarisAuthenticationParameters implements IcebergCatalogP @JsonProperty(value = "authenticationType") private final AuthenticationType authenticationType; - public PolarisAuthenticationParameters( + public AuthenticationParametersDpo( @JsonProperty(value = "authenticationType", required = true) @Nonnull AuthenticationType authenticationType) { this.authenticationType = authenticationType; @@ -53,16 +53,16 @@ public PolarisAuthenticationParameters( public abstract AuthenticationParameters asAuthenticationParametersModel(); - public static PolarisAuthenticationParameters fromAuthenticationParametersModelWithSecrets( + public static AuthenticationParametersDpo fromAuthenticationParametersModelWithSecrets( AuthenticationParameters authenticationParameters, Map secretReferences) { - PolarisAuthenticationParameters config = null; + AuthenticationParametersDpo config = null; switch (authenticationParameters.getAuthenticationType()) { case OAUTH: OAuthClientCredentialsParameters oauthClientCredentialsModel = (OAuthClientCredentialsParameters) authenticationParameters; config = - new PolarisOAuthClientCredentialsParameters( + new OAuthClientCredentialsParametersDpo( AuthenticationType.OAUTH, oauthClientCredentialsModel.getTokenUri(), oauthClientCredentialsModel.getClientId(), @@ -73,7 +73,7 @@ public static PolarisAuthenticationParameters fromAuthenticationParametersModelW BearerAuthenticationParameters bearerAuthenticationParametersModel = (BearerAuthenticationParameters) authenticationParameters; config = - new PolarisBearerAuthenticationParameters( + new BearerAuthenticationParametersDpo( AuthenticationType.BEARER, secretReferences.get(INLINE_BEARER_TOKEN_REFERENCE_KEY)); break; default: diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java similarity index 94% rename from polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java rename to polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java index c3d9ec3002..3499e4cb30 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisBearerAuthenticationParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java @@ -28,12 +28,12 @@ import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.secrets.UserSecretsManager; -public class PolarisBearerAuthenticationParameters extends PolarisAuthenticationParameters { +public class BearerAuthenticationParametersDpo extends AuthenticationParametersDpo { @JsonProperty(value = "bearerTokenReference") private final UserSecretReference bearerTokenReference; - public PolarisBearerAuthenticationParameters( + public BearerAuthenticationParametersDpo( @JsonProperty(value = "authenticationType", required = true) @Nonnull AuthenticationType authenticationType, @JsonProperty(value = "bearerTokenReference", required = true) @Nonnull diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java similarity index 85% rename from polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java rename to polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java index a6ba2d5f48..69a0dfedbc 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java @@ -36,12 +36,10 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "connectionType", visible = true) @JsonSubTypes({ - @JsonSubTypes.Type(value = IcebergRestConnectionConfigurationInfo.class, name = "ICEBERG_REST"), + @JsonSubTypes.Type(value = IcebergRestConnectionConfigInfoDpo.class, name = "ICEBERG_REST"), }) -public abstract class PolarisConnectionConfigurationInfo - implements IcebergCatalogPropertiesProvider { - private static final Logger logger = - LoggerFactory.getLogger(PolarisConnectionConfigurationInfo.class); +public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertiesProvider { + private static final Logger logger = LoggerFactory.getLogger(ConnectionConfigInfoDpo.class); // The type of the connection private final ConnectionType connectionType; @@ -49,14 +47,14 @@ public abstract class PolarisConnectionConfigurationInfo // The URI of the remote catalog private final String uri; - public PolarisConnectionConfigurationInfo( + public ConnectionConfigInfoDpo( @JsonProperty(value = "connectionType", required = true) @Nonnull ConnectionType connectionType, @JsonProperty(value = "uri", required = true) @Nonnull String uri) { this(connectionType, uri, true); } - protected PolarisConnectionConfigurationInfo( + protected ConnectionConfigInfoDpo( ConnectionType connectionType, String uri, boolean validateUri) { this.connectionType = connectionType; this.uri = uri; @@ -89,10 +87,10 @@ public String serialize() { } } - public static PolarisConnectionConfigurationInfo deserialize( + public static ConnectionConfigInfoDpo deserialize( @Nonnull PolarisDiagnostics diagnostics, final @Nonnull String jsonStr) { try { - return DEFAULT_MAPPER.readValue(jsonStr, PolarisConnectionConfigurationInfo.class); + return DEFAULT_MAPPER.readValue(jsonStr, ConnectionConfigInfoDpo.class); } catch (JsonProcessingException exception) { diagnostics.fail( "fail_to_deserialize_connection_configuration", exception, "jsonStr={}", jsonStr); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java similarity index 90% rename from polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java rename to polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java index b7c8609333..690ac6d1d1 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java @@ -29,21 +29,21 @@ import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; import org.apache.polaris.core.secrets.UserSecretsManager; -public class IcebergRestConnectionConfigurationInfo extends PolarisConnectionConfigurationInfo +public class IcebergRestConnectionConfigInfoDpo extends ConnectionConfigInfoDpo implements IcebergCatalogPropertiesProvider { private final String remoteCatalogName; - private final PolarisAuthenticationParameters authenticationParameters; + private final AuthenticationParametersDpo authenticationParameters; - public IcebergRestConnectionConfigurationInfo( + public IcebergRestConnectionConfigInfoDpo( @JsonProperty(value = "connectionType", required = true) @Nonnull ConnectionType connectionType, @JsonProperty(value = "uri", required = true) @Nonnull String uri, @JsonProperty(value = "remoteCatalogName", required = false) @Nullable String remoteCatalogName, @JsonProperty(value = "authenticationParameters", required = false) @Nonnull - PolarisAuthenticationParameters authenticationParameters) { + AuthenticationParametersDpo authenticationParameters) { super(connectionType, uri); this.remoteCatalogName = remoteCatalogName; this.authenticationParameters = authenticationParameters; @@ -53,7 +53,7 @@ public String getRemoteCatalogName() { return remoteCatalogName; } - public PolarisAuthenticationParameters getAuthenticationParameters() { + public AuthenticationParametersDpo getAuthenticationParameters() { return authenticationParameters; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java similarity index 97% rename from polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java rename to polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java index c7f1bc593d..f0e9f1ddd5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/PolarisOAuthClientCredentialsParameters.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java @@ -38,7 +38,7 @@ import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.secrets.UserSecretsManager; -public class PolarisOAuthClientCredentialsParameters extends PolarisAuthenticationParameters { +public class OAuthClientCredentialsParametersDpo extends AuthenticationParametersDpo { private static final Joiner COLON_JOINER = Joiner.on(":"); @@ -54,7 +54,7 @@ public class PolarisOAuthClientCredentialsParameters extends PolarisAuthenticati @JsonProperty(value = "scopes") private final List scopes; - public PolarisOAuthClientCredentialsParameters( + public OAuthClientCredentialsParametersDpo( @JsonProperty(value = "authenticationType", required = true) @Nonnull AuthenticationType authenticationType, @JsonProperty(value = "tokenUri", required = false) @Nullable String tokenUri, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index 5ee19baf13..a6a80c19e3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -43,10 +43,10 @@ import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.config.BehaviorChangeConfiguration; +import org.apache.polaris.core.connection.AuthenticationParametersDpo; +import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; import org.apache.polaris.core.connection.ConnectionType; -import org.apache.polaris.core.connection.IcebergRestConnectionConfigurationInfo; -import org.apache.polaris.core.connection.PolarisAuthenticationParameters; -import org.apache.polaris.core.connection.PolarisConnectionConfigurationInfo; +import org.apache.polaris.core.connection.IcebergRestConnectionConfigInfoDpo; import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.storage.FileStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; @@ -176,7 +176,7 @@ private StorageConfigInfo getStorageInfo(Map internalProperties) private ConnectionConfigInfo getConnectionInfo(Map internalProperties) { if (internalProperties.containsKey( PolarisEntityConstants.getConnectionConfigInfoPropertyName())) { - PolarisConnectionConfigurationInfo configInfo = getConnectionConfigurationInfo(); + ConnectionConfigInfoDpo configInfo = getConnectionConfigurationInfo(); return configInfo.asConnectionConfigInfoModel(); } return null; @@ -211,13 +211,12 @@ public boolean isPassthroughFacade() { .containsKey(PolarisEntityConstants.getConnectionConfigInfoPropertyName()); } - public PolarisConnectionConfigurationInfo getConnectionConfigurationInfo() { + public ConnectionConfigInfoDpo getConnectionConfigurationInfo() { String configStr = getInternalPropertiesAsMap() .get(PolarisEntityConstants.getConnectionConfigInfoPropertyName()); if (configStr != null) { - return PolarisConnectionConfigurationInfo.deserialize( - new PolarisDefaultDiagServiceImpl(), configStr); + return ConnectionConfigInfoDpo.deserialize(new PolarisDefaultDiagServiceImpl(), configStr); } return null; } @@ -326,16 +325,16 @@ public Builder setConnectionConfigurationInfoWithSecrets( ConnectionConfigInfo connectionConfigurationModel, Map secretReferences) { if (connectionConfigurationModel != null) { - PolarisConnectionConfigurationInfo config; + ConnectionConfigInfoDpo config; switch (connectionConfigurationModel.getConnectionType()) { case ICEBERG_REST: IcebergRestConnectionConfigInfo icebergRestConfigModel = (IcebergRestConnectionConfigInfo) connectionConfigurationModel; - PolarisAuthenticationParameters authenticationParameters = - PolarisAuthenticationParameters.fromAuthenticationParametersModelWithSecrets( + AuthenticationParametersDpo authenticationParameters = + AuthenticationParametersDpo.fromAuthenticationParametersModelWithSecrets( icebergRestConfigModel.getAuthenticationParameters(), secretReferences); config = - new IcebergRestConnectionConfigurationInfo( + new IcebergRestConnectionConfigInfoDpo( ConnectionType.ICEBERG_REST, icebergRestConfigModel.getUri(), icebergRestConfigModel.getRemoteCatalogName(), diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java similarity index 91% rename from polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java rename to polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java index 03c5220cb4..b2e7ac99c6 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/connection/PolarisConnectionConfigurationInfoTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class PolarisConnectionConfigurationInfoTest { +public class ConnectionConfigInfoDpoTest { PolarisDiagnostics polarisDiagnostics = new PolarisDefaultDiagServiceImpl(); ObjectMapper objectMapper = new ObjectMapper(); @@ -52,8 +52,8 @@ void testOAuthClientCredentialsParameters() throws JsonProcessingException { + " \"scopes\": [\"PRINCIPAL_ROLE:ALL\"]" + " }" + "}"; - PolarisConnectionConfigurationInfo connectionConfigurationInfo = - PolarisConnectionConfigurationInfo.deserialize(polarisDiagnostics, json); + ConnectionConfigInfoDpo connectionConfigurationInfo = + ConnectionConfigInfoDpo.deserialize(polarisDiagnostics, json); Assertions.assertNotNull(connectionConfigurationInfo); System.out.println(connectionConfigurationInfo.serialize()); JsonNode tree1 = objectMapper.readTree(json); @@ -80,8 +80,8 @@ void testBearerAuthenticationParameters() throws JsonProcessingException { + " }" + " }" + "}"; - PolarisConnectionConfigurationInfo connectionConfigurationInfo = - PolarisConnectionConfigurationInfo.deserialize(polarisDiagnostics, json); + ConnectionConfigInfoDpo connectionConfigurationInfo = + ConnectionConfigInfoDpo.deserialize(polarisDiagnostics, json); Assertions.assertNotNull(connectionConfigurationInfo); System.out.println(connectionConfigurationInfo.serialize()); JsonNode tree1 = objectMapper.readTree(json); diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 28189d2d08..893df5237c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -74,7 +74,7 @@ import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.config.FeatureConfiguration; -import org.apache.polaris.core.connection.PolarisAuthenticationParameters; +import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.CatalogRoleEntity; @@ -612,8 +612,7 @@ private Map extractSecretReferences( UserSecretReference secretReference = secretsManager.writeSecret(inlineClientSecret, forEntity); secretReferences.put( - PolarisAuthenticationParameters.INLINE_CLIENT_SECRET_REFERENCE_KEY, - secretReference); + AuthenticationParametersDpo.INLINE_CLIENT_SECRET_REFERENCE_KEY, secretReference); break; } case BEARER: @@ -624,8 +623,7 @@ private Map extractSecretReferences( UserSecretReference secretReference = secretsManager.writeSecret(inlineBearerToken, forEntity); secretReferences.put( - PolarisAuthenticationParameters.INLINE_BEARER_TOKEN_REFERENCE_KEY, - secretReference); + AuthenticationParametersDpo.INLINE_BEARER_TOKEN_REFERENCE_KEY, secretReference); break; } default: diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 0ab9f64b17..63413ebd98 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -74,8 +74,8 @@ import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.PolarisConfigurationStore; -import org.apache.polaris.core.connection.IcebergRestConnectionConfigurationInfo; -import org.apache.polaris.core.connection.PolarisConnectionConfigurationInfo; +import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; +import org.apache.polaris.core.connection.IcebergRestConnectionConfigInfoDpo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; @@ -168,7 +168,7 @@ private UserSecretsManager getUserSecretsManager() { protected void initializeCatalog() { CatalogEntity resolvedCatalogEntity = CatalogEntity.of(resolutionManifest.getResolvedReferenceCatalogEntity().getRawLeafEntity()); - PolarisConnectionConfigurationInfo connectionConfigurationInfo = + ConnectionConfigInfoDpo connectionConfigurationInfo = resolvedCatalogEntity.getConnectionConfigurationInfo(); if (connectionConfigurationInfo != null) { LOGGER @@ -188,7 +188,7 @@ protected void initializeCatalog() { .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) .build()); federatedCatalog.initialize( - ((IcebergRestConnectionConfigurationInfo) connectionConfigurationInfo) + ((IcebergRestConnectionConfigInfoDpo) connectionConfigurationInfo) .getRemoteCatalogName(), connectionConfigurationInfo.asIcebergCatalogProperties(getUserSecretsManager())); break; From fd7ae593b002ba31fe219bd477d7c6759557ab1d Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Thu, 3 Apr 2025 23:04:12 -0700 Subject: [PATCH 12/17] Use UserSecretsManagerFactory to Produce the UserSecretsManager (#1) * Move PolarisAuthenticationParameters to a top-level property according to the latest spec * Create a Factory for UserSecretsManager * Fix a typo in UnsafeInMemorySecretsManagerFactory --- .../connection/ConnectionConfigInfoDpo.java | 47 +++++++++++++++++-- .../IcebergRestConnectionConfigInfoDpo.java | 20 +++----- .../OAuthClientCredentialsParametersDpo.java | 1 + .../polaris/core/entity/CatalogEntity.java | 26 ++-------- .../secrets/UnsafeInMemorySecretsManager.java | 9 ++-- .../core/secrets/UserSecretsManager.java | 9 ++-- .../secrets/UserSecretsManagerFactory.java | 31 ++++++++++++ .../main/resources/application-it.properties | 2 + .../src/main/resources/application.properties | 2 + .../quarkus/config/QuarkusProducers.java | 17 +++++++ .../QuarkusSecretsManagerConfiguration.java | 33 +++++++++++++ .../admin/PolarisAdminServiceAuthzTest.java | 1 + .../quarkus/admin/PolarisAuthzTestBase.java | 16 ++++++- .../catalog/GenericTableCatalogTest.java | 6 +++ .../IcebergCatalogHandlerAuthzTest.java | 4 ++ .../quarkus/catalog/IcebergCatalogTest.java | 6 +++ .../catalog/IcebergCatalogViewTest.java | 6 +++ .../service/admin/PolarisAdminService.java | 7 +-- .../service/admin/PolarisServiceImpl.java | 14 +++++- .../iceberg/IcebergCatalogAdapter.java | 5 ++ .../iceberg/IcebergCatalogHandler.java | 7 +-- .../PolarisCallContextCatalogFactory.java | 4 ++ .../UnsafeInMemorySecretsManagerFactory.java | 40 ++++++++++++++++ .../apache/polaris/service/TestServices.java | 20 +++++++- 24 files changed, 277 insertions(+), 56 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManagerFactory.java create mode 100644 quarkus/service/src/main/java/org/apache/polaris/service/quarkus/secrets/QuarkusSecretsManagerConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/secrets/UnsafeInMemorySecretsManagerFactory.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java index 69a0dfedbc..d4bd48942d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java @@ -29,8 +29,11 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.util.Map; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.ConnectionConfigInfo; +import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; +import org.apache.polaris.core.secrets.UserSecretReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,17 +50,26 @@ public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertie // The URI of the remote catalog private final String uri; + // The authentication parameters for the connection + private final AuthenticationParametersDpo authenticationParameters; + public ConnectionConfigInfoDpo( @JsonProperty(value = "connectionType", required = true) @Nonnull ConnectionType connectionType, - @JsonProperty(value = "uri", required = true) @Nonnull String uri) { - this(connectionType, uri, true); + @JsonProperty(value = "uri", required = true) @Nonnull String uri, + @JsonProperty(value = "authenticationParameters", required = true) @Nonnull + AuthenticationParametersDpo authenticationParameters) { + this(connectionType, uri, authenticationParameters, true); } protected ConnectionConfigInfoDpo( - ConnectionType connectionType, String uri, boolean validateUri) { + @Nonnull ConnectionType connectionType, + @Nonnull String uri, + @Nonnull AuthenticationParametersDpo authenticationParameters, + boolean validateUri) { this.connectionType = connectionType; this.uri = uri; + this.authenticationParameters = authenticationParameters; if (validateUri) { validateUri(uri); } @@ -71,6 +83,10 @@ public String getUri() { return uri; } + public AuthenticationParametersDpo getAuthenticationParameters() { + return authenticationParameters; + } + private static final ObjectMapper DEFAULT_MAPPER; static { @@ -108,5 +124,30 @@ protected void validateUri(String uri) { } } + public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( + ConnectionConfigInfo connectionConfigurationModel, + Map secretReferences) { + ConnectionConfigInfoDpo config = null; + switch (connectionConfigurationModel.getConnectionType()) { + case ICEBERG_REST: + IcebergRestConnectionConfigInfo icebergRestConfigModel = + (IcebergRestConnectionConfigInfo) connectionConfigurationModel; + AuthenticationParametersDpo authenticationParameters = + AuthenticationParametersDpo.fromAuthenticationParametersModelWithSecrets( + icebergRestConfigModel.getAuthenticationParameters(), secretReferences); + config = + new IcebergRestConnectionConfigInfoDpo( + ConnectionType.ICEBERG_REST, + icebergRestConfigModel.getUri(), + authenticationParameters, + icebergRestConfigModel.getRemoteCatalogName()); + break; + default: + throw new IllegalStateException( + "Unsupported connection type: " + connectionConfigurationModel.getConnectionType()); + } + return config; + } + public abstract ConnectionConfigInfo asConnectionConfigInfoModel(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java index 690ac6d1d1..b138370726 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java @@ -34,29 +34,22 @@ public class IcebergRestConnectionConfigInfoDpo extends ConnectionConfigInfoDpo private final String remoteCatalogName; - private final AuthenticationParametersDpo authenticationParameters; - public IcebergRestConnectionConfigInfoDpo( @JsonProperty(value = "connectionType", required = true) @Nonnull ConnectionType connectionType, @JsonProperty(value = "uri", required = true) @Nonnull String uri, + @JsonProperty(value = "authenticationParameters", required = true) @Nonnull + AuthenticationParametersDpo authenticationParameters, @JsonProperty(value = "remoteCatalogName", required = false) @Nullable - String remoteCatalogName, - @JsonProperty(value = "authenticationParameters", required = false) @Nonnull - AuthenticationParametersDpo authenticationParameters) { - super(connectionType, uri); + String remoteCatalogName) { + super(connectionType, uri, authenticationParameters); this.remoteCatalogName = remoteCatalogName; - this.authenticationParameters = authenticationParameters; } public String getRemoteCatalogName() { return remoteCatalogName; } - public AuthenticationParametersDpo getAuthenticationParameters() { - return authenticationParameters; - } - @Override public @Nonnull Map asIcebergCatalogProperties( UserSecretsManager secretsManager) { @@ -65,7 +58,7 @@ public AuthenticationParametersDpo getAuthenticationParameters() { if (getRemoteCatalogName() != null) { properties.put(CatalogProperties.WAREHOUSE_LOCATION, getRemoteCatalogName()); } - properties.putAll(authenticationParameters.asIcebergCatalogProperties(secretsManager)); + properties.putAll(getAuthenticationParameters().asIcebergCatalogProperties(secretsManager)); return properties; } @@ -75,7 +68,8 @@ public ConnectionConfigInfo asConnectionConfigInfoModel() { .setConnectionType(ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) .setUri(getUri()) .setRemoteCatalogName(getRemoteCatalogName()) - .setAuthenticationParameters(authenticationParameters.asAuthenticationParametersModel()) + .setAuthenticationParameters( + getAuthenticationParameters().asAuthenticationParametersModel()) .build(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java index f0e9f1ddd5..e80a8467d3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java @@ -94,6 +94,7 @@ public OAuthClientCredentialsParametersDpo( Objects.requireNonNullElse(scopes, List.of(OAuth2Properties.CATALOG_SCOPE))); } + @JsonIgnore private @Nonnull String getCredentialAsConcatenatedString(UserSecretsManager secretsManager) { String clientSecret = secretsManager.readSecret(getClientSecretReference()); return COLON_JOINER.join(clientId, clientSecret); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index a6a80c19e3..ff4ce894a0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -39,14 +39,10 @@ import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; -import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.config.BehaviorChangeConfiguration; -import org.apache.polaris.core.connection.AuthenticationParametersDpo; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; -import org.apache.polaris.core.connection.ConnectionType; -import org.apache.polaris.core.connection.IcebergRestConnectionConfigInfoDpo; import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.storage.FileStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; @@ -325,25 +321,9 @@ public Builder setConnectionConfigurationInfoWithSecrets( ConnectionConfigInfo connectionConfigurationModel, Map secretReferences) { if (connectionConfigurationModel != null) { - ConnectionConfigInfoDpo config; - switch (connectionConfigurationModel.getConnectionType()) { - case ICEBERG_REST: - IcebergRestConnectionConfigInfo icebergRestConfigModel = - (IcebergRestConnectionConfigInfo) connectionConfigurationModel; - AuthenticationParametersDpo authenticationParameters = - AuthenticationParametersDpo.fromAuthenticationParametersModelWithSecrets( - icebergRestConfigModel.getAuthenticationParameters(), secretReferences); - config = - new IcebergRestConnectionConfigInfoDpo( - ConnectionType.ICEBERG_REST, - icebergRestConfigModel.getUri(), - icebergRestConfigModel.getRemoteCatalogName(), - authenticationParameters); - break; - default: - throw new IllegalStateException( - "Unsupported connection type: " + connectionConfigurationModel.getConnectionType()); - } + ConnectionConfigInfoDpo config = + ConnectionConfigInfoDpo.fromConnectionConfigInfoModelWithSecrets( + connectionConfigurationModel, secretReferences); internalProperties.put( PolarisEntityConstants.getConnectionConfigInfoPropertyName(), config.serialize()); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java index d5cdfa0c3f..fcf255ee31 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.core.secrets; +import jakarta.annotation.Nonnull; import java.io.UnsupportedEncodingException; import java.security.SecureRandom; import java.util.Base64; @@ -46,7 +47,8 @@ public class UnsafeInMemorySecretsManager implements UserSecretsManager { /** {@inheritDoc} */ @Override - public UserSecretReference writeSecret(String secret, PolarisEntity forEntity) { + @Nonnull + public UserSecretReference writeSecret(@Nonnull String secret, @Nonnull PolarisEntity forEntity) { // For illustrative purposes and to exercise the control flow of requiring both the stored // secret as well as the secretReferencePayload to recover the original secret, we'll use // basic XOR encryption and store the randomly generated key in the reference payload. @@ -107,7 +109,8 @@ public UserSecretReference writeSecret(String secret, PolarisEntity forEntity) { /** {@inheritDoc} */ @Override - public String readSecret(UserSecretReference secretReference) { + @Nonnull + public String readSecret(@Nonnull UserSecretReference secretReference) { // TODO: Precondition checks and/or wire in PolarisDiagnostics String encryptedSecretCipherTextBase64 = rawSecretStore.get(secretReference.getUrn()); if (encryptedSecretCipherTextBase64 == null) { @@ -148,7 +151,7 @@ public String readSecret(UserSecretReference secretReference) { /** {@inheritDoc} */ @Override - public void deleteSecret(UserSecretReference secretReference) { + public void deleteSecret(@Nonnull UserSecretReference secretReference) { rawSecretStore.remove(secretReference.getUrn()); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java index 1dfd11d213..8df656e4ee 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.core.secrets; +import jakarta.annotation.Nonnull; import org.apache.polaris.core.entity.PolarisEntity; /** @@ -42,7 +43,8 @@ public interface UserSecretsManager { * @return A reference object that can be used to retrieve the secret which is safe to store in * its entirety within a persisted PolarisEntity */ - UserSecretReference writeSecret(String secret, PolarisEntity forEntity); + @Nonnull + UserSecretReference writeSecret(@Nonnull String secret, @Nonnull PolarisEntity forEntity); /** * Retrieve a secret using the {@code secretReference}. @@ -50,12 +52,13 @@ public interface UserSecretsManager { * @param secretReference Identifier and any associated payload used for retrieving the secret * @return The stored secret, or null if it no longer exists */ - String readSecret(UserSecretReference secretReference); + @Nonnull + String readSecret(@Nonnull UserSecretReference secretReference); /** * Delete a stored secret * * @param secretReference Identifier and any associated payload used for retrieving the secret */ - void deleteSecret(UserSecretReference secretReference); + void deleteSecret(@Nonnull UserSecretReference secretReference); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManagerFactory.java new file mode 100644 index 0000000000..a8ee2b7b44 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManagerFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.secrets; + +import org.apache.polaris.core.context.RealmContext; + +/** + * Factory for creating {@link UserSecretsManager} instances. + * + *

Each {@link UserSecretsManager} instance is associated with a {@link RealmContext} and is + * responsible for managing the secrets for the user in that realm. + */ +public interface UserSecretsManagerFactory { + UserSecretsManager getOrCreateUserSecretsManager(RealmContext realmContext); +} diff --git a/quarkus/defaults/src/main/resources/application-it.properties b/quarkus/defaults/src/main/resources/application-it.properties index 5e110071de..906d13c1ed 100644 --- a/quarkus/defaults/src/main/resources/application-it.properties +++ b/quarkus/defaults/src/main/resources/application-it.properties @@ -29,6 +29,8 @@ quarkus.management.port=0 # polaris.persistence.type=in-memory-atomic polaris.persistence.type=in-memory +polaris.secrets-manager.type=in-memory + polaris.features.defaults."ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING"=false polaris.features.defaults."ALLOW_EXTERNAL_METADATA_FILE_LOCATION"=false polaris.features.defaults."ALLOW_OVERLAPPING_CATALOG_URLS"=true diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index 0cd79eadae..d6f579910b 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -99,6 +99,8 @@ polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE", # polaris.persistence.type=in-memory-atomic polaris.persistence.type=in-memory +polaris.secrets-manager.type=in-memory + polaris.file-io.type=default polaris.log.request-id-header-name=Polaris-Request-Id diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index 16b7439791..e4c34c4652 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -45,6 +45,8 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.auth.ActiveRolesProvider; import org.apache.polaris.service.auth.Authenticator; @@ -62,6 +64,7 @@ import org.apache.polaris.service.quarkus.persistence.QuarkusPersistenceConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusRateLimiterFilterConfiguration; import org.apache.polaris.service.quarkus.ratelimiter.QuarkusTokenBucketConfiguration; +import org.apache.polaris.service.quarkus.secrets.QuarkusSecretsManagerConfiguration; import org.apache.polaris.service.ratelimiter.RateLimiter; import org.apache.polaris.service.ratelimiter.TokenBucketFactory; import org.apache.polaris.service.task.TaskHandlerConfiguration; @@ -150,6 +153,13 @@ public MetaStoreManagerFactory metaStoreManagerFactory( return metaStoreManagerFactories.select(Identifier.Literal.of(config.type())).get(); } + @Produces + public UserSecretsManagerFactory userSecretsManagerFactory( + QuarkusSecretsManagerConfiguration config, + @Any Instance userSecretsManagerFactories) { + return userSecretsManagerFactories.select(Identifier.Literal.of(config.type())).get(); + } + /** * Eagerly initialize the in-memory default realm on startup, so that users can check the * credentials printed to stdout immediately. @@ -218,6 +228,13 @@ public PolarisMetaStoreManager polarisMetaStoreManager( return metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); } + @Produces + @RequestScoped + public UserSecretsManager userSecretsManager( + RealmContext realmContext, UserSecretsManagerFactory userSecretsManagerFactory) { + return userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); + } + @Produces @RequestScoped public BasePersistence polarisMetaStoreSession( diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/secrets/QuarkusSecretsManagerConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/secrets/QuarkusSecretsManagerConfiguration.java new file mode 100644 index 0000000000..a472dc681f --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/secrets/QuarkusSecretsManagerConfiguration.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.secrets; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.secrets-manager") +public interface QuarkusSecretsManagerConfiguration { + + /** + * The type of the UserSecretsManagerFactory to use. This is the {@link + * org.apache.polaris.core.secrets.UserSecretsManagerFactory} identifier. + */ + String type(); +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java index 7199f4fa7e..55b2d9f341 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAdminServiceAuthzTest.java @@ -53,6 +53,7 @@ private PolarisAdminService newTestAdminService(Set activatedPrincipalRo callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedPrincipal, activatedPrincipalRoles), polarisAuthorizer); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index d908752812..4b0037eed0 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -75,6 +75,8 @@ import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.generic.GenericTableCatalog; @@ -176,6 +178,7 @@ public Map getConfigOverrides() { @Inject protected MetaStoreManagerFactory managerFactory; @Inject protected RealmEntityManagerFactory realmEntityManagerFactory; @Inject protected CallContextCatalogFactory callContextCatalogFactory; + @Inject protected UserSecretsManagerFactory userSecretsManagerFactory; @Inject protected PolarisDiagnostics diagServices; @Inject protected Clock clock; @Inject protected FileIOFactory fileIOFactory; @@ -185,6 +188,7 @@ public Map getConfigOverrides() { protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; protected PolarisMetaStoreManager metaStoreManager; + protected UserSecretsManager userSecretsManager; protected TransactionalPersistence metaStoreSession; protected PolarisBaseEntity catalogEntity; protected PrincipalEntity principalEntity; @@ -205,6 +209,7 @@ public static void setUpMocks() { public void before(TestInfo testInfo) { RealmContext realmContext = testInfo::getDisplayName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); Map configMap = Map.of( @@ -248,6 +253,7 @@ public void before(TestInfo testInfo) { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedRoot, Set.of()), polarisAuthorizer); @@ -465,16 +471,22 @@ public static class TestPolarisCallContextCatalogFactory extends PolarisCallContextCatalogFactory { public TestPolarisCallContextCatalogFactory() { - super(null, null, null, null); + super(null, null, null, null, null); } @Inject public TestPolarisCallContextCatalogFactory( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, + UserSecretsManagerFactory userSecretsManagerFactory, TaskExecutor taskExecutor, FileIOFactory fileIOFactory) { - super(entityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); + super( + entityManagerFactory, + metaStoreManagerFactory, + userSecretsManagerFactory, + taskExecutor, + fileIOFactory); } @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java index 311c239902..462cc7bb73 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java @@ -67,6 +67,8 @@ import org.apache.polaris.core.persistence.dao.entity.BaseResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; @@ -119,6 +121,7 @@ public Map getConfigOverrides() { public static final String SESSION_TOKEN = "session_token"; @Inject MetaStoreManagerFactory managerFactory; + @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject PolarisDiagnostics diagServices; @@ -129,6 +132,7 @@ public Map getConfigOverrides() { private AwsStorageConfigInfo storageConfigModel; private String realmName; private PolarisMetaStoreManager metaStoreManager; + private UserSecretsManager userSecretsManager; private PolarisCallContext polarisContext; private PolarisAdminService adminService; private PolarisEntityManager entityManager; @@ -158,6 +162,7 @@ public void before(TestInfo testInfo) { testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -192,6 +197,7 @@ public void before(TestInfo testInfo) { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index b8742404b6..11a938f316 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -111,6 +111,7 @@ private IcebergCatalogHandler newWrapper( callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedPrincipal, activatedPrincipalRoles), factory, catalogName, @@ -249,6 +250,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedPrincipal, Set.of(PRINCIPAL_ROLE1, PRINCIPAL_ROLE2)), callContextCatalogFactory, CATALOG_NAME, @@ -281,6 +283,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext(authenticatedPrincipal1, Set.of(PRINCIPAL_ROLE1, PRINCIPAL_ROLE2)), callContextCatalogFactory, CATALOG_NAME, @@ -1802,6 +1805,7 @@ public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) } }, managerFactory, + userSecretsManagerFactory, Mockito.mock(), new DefaultFileIOFactory( realmEntityManagerFactory, managerFactory, new PolarisConfigurationStore() {})) { diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index eedc9a588d..690a335a2b 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -99,6 +99,8 @@ import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -173,12 +175,14 @@ public Map getConfigOverrides() { @Inject MetaStoreManagerFactory managerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisDiagnostics diagServices; private IcebergCatalog catalog; private CallContext callContext; private String realmName; private PolarisMetaStoreManager metaStoreManager; + private UserSecretsManager userSecretsManager; private PolarisCallContext polarisContext; private PolarisAdminService adminService; private PolarisEntityManager entityManager; @@ -205,6 +209,7 @@ public void before(TestInfo testInfo) { testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -240,6 +245,7 @@ public void before(TestInfo testInfo) { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index fa51a021d8..8d7708dd73 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -58,6 +58,8 @@ import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; @@ -98,6 +100,7 @@ public Map getConfigOverrides() { public static final String CATALOG_NAME = "polaris-catalog"; @Inject MetaStoreManagerFactory managerFactory; + @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisDiagnostics diagServices; @@ -105,6 +108,7 @@ public Map getConfigOverrides() { private String realmName; private PolarisMetaStoreManager metaStoreManager; + private UserSecretsManager userSecretsManager; private PolarisCallContext polarisContext; @BeforeAll @@ -131,6 +135,7 @@ public void before(TestInfo testInfo) { RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), @@ -167,6 +172,7 @@ public void before(TestInfo testInfo) { callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); adminService.createCatalog( diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 893df5237c..133249a0f6 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -100,7 +100,6 @@ import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; -import org.apache.polaris.core.secrets.UnsafeInMemorySecretsManager; import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; @@ -129,6 +128,7 @@ public class PolarisAdminService { private final AuthenticatedPolarisPrincipal authenticatedPrincipal; private final PolarisAuthorizer authorizer; private final PolarisMetaStoreManager metaStoreManager; + private final UserSecretsManager userSecretsManager; // Initialized in the authorize methods. private PolarisResolutionManifest resolutionManifest = null; @@ -137,6 +137,7 @@ public PolarisAdminService( @NotNull CallContext callContext, @NotNull PolarisEntityManager entityManager, @NotNull PolarisMetaStoreManager metaStoreManager, + @NotNull UserSecretsManager userSecretsManager, @NotNull SecurityContext securityContext, @NotNull PolarisAuthorizer authorizer) { this.callContext = callContext; @@ -154,6 +155,7 @@ public PolarisAdminService( this.authenticatedPrincipal = (AuthenticatedPolarisPrincipal) securityContext.getUserPrincipal(); this.authorizer = authorizer; + this.userSecretsManager = userSecretsManager; } private PolarisCallContext getCurrentPolarisContext() { @@ -161,8 +163,7 @@ private PolarisCallContext getCurrentPolarisContext() { } private UserSecretsManager getUserSecretsManager() { - // TODO: Wire into appropriate factories and/or contexts. - return UnsafeInMemorySecretsManager.GLOBAL_INSTANCE; + return userSecretsManager; } private Optional findCatalogByName(String name) { diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index b47d50208f..c2c53c17b2 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -68,6 +68,8 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.admin.api.PolarisCatalogsApiService; import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApiService; import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; @@ -85,16 +87,19 @@ public class PolarisServiceImpl private final RealmEntityManagerFactory entityManagerFactory; private final PolarisAuthorizer polarisAuthorizer; private final MetaStoreManagerFactory metaStoreManagerFactory; + private final UserSecretsManagerFactory userSecretsManagerFactory; private final CallContext callContext; @Inject public PolarisServiceImpl( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, + UserSecretsManagerFactory userSecretsManagerFactory, PolarisAuthorizer polarisAuthorizer, CallContext callContext) { this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; + this.userSecretsManagerFactory = userSecretsManagerFactory; this.polarisAuthorizer = polarisAuthorizer; this.callContext = callContext; // FIXME: This is a hack to set the current context for downstream calls. @@ -113,8 +118,15 @@ private PolarisAdminService newAdminService( entityManagerFactory.getOrCreateEntityManager(realmContext); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + UserSecretsManager userSecretsManager = + userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); return new PolarisAdminService( - callContext, entityManager, metaStoreManager, securityContext, polarisAuthorizer); + callContext, + entityManager, + metaStoreManager, + userSecretsManager, + securityContext, + polarisAuthorizer); } /** From PolarisCatalogsApiService */ diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index ccb6b44f57..21316536a0 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -67,6 +67,7 @@ import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.service.catalog.AccessDelegationMode; import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; @@ -132,6 +133,7 @@ public class IcebergCatalogAdapter private final CallContextCatalogFactory catalogFactory; private final PolarisEntityManager entityManager; private final PolarisMetaStoreManager metaStoreManager; + private final UserSecretsManager userSecretsManager; private final PolarisAuthorizer polarisAuthorizer; private final CatalogPrefixParser prefixParser; @@ -142,6 +144,7 @@ public IcebergCatalogAdapter( CallContextCatalogFactory catalogFactory, PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, + UserSecretsManager userSecretsManager, PolarisAuthorizer polarisAuthorizer, CatalogPrefixParser prefixParser) { this.realmContext = realmContext; @@ -149,6 +152,7 @@ public IcebergCatalogAdapter( this.catalogFactory = catalogFactory; this.entityManager = entityManager; this.metaStoreManager = metaStoreManager; + this.userSecretsManager = userSecretsManager; this.polarisAuthorizer = polarisAuthorizer; this.prefixParser = prefixParser; @@ -188,6 +192,7 @@ private IcebergCatalogHandler newHandlerWrapper( callContext, entityManager, metaStoreManager, + userSecretsManager, securityContext, catalogFactory, catalogName, diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 63413ebd98..e8f46e15ae 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -86,7 +86,6 @@ import org.apache.polaris.core.persistence.TransactionWorkspaceMetaStoreManager; import org.apache.polaris.core.persistence.dao.entity.EntitiesResult; import org.apache.polaris.core.persistence.dao.entity.EntityWithPath; -import org.apache.polaris.core.secrets.UnsafeInMemorySecretsManager; import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.service.catalog.SupportsNotifications; @@ -117,6 +116,7 @@ public class IcebergCatalogHandler extends CatalogHandler implements AutoCloseab private static final Logger LOGGER = LoggerFactory.getLogger(IcebergCatalogHandler.class); private final PolarisMetaStoreManager metaStoreManager; + private final UserSecretsManager userSecretsManager; private final CallContextCatalogFactory catalogFactory; // Catalog instance will be initialized after authorizing resolver successfully resolves @@ -129,12 +129,14 @@ public IcebergCatalogHandler( CallContext callContext, PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, + UserSecretsManager userSecretsManager, SecurityContext securityContext, CallContextCatalogFactory catalogFactory, String catalogName, PolarisAuthorizer authorizer) { super(callContext, entityManager, securityContext, catalogName, authorizer); this.metaStoreManager = metaStoreManager; + this.userSecretsManager = userSecretsManager; this.catalogFactory = catalogFactory; } @@ -160,8 +162,7 @@ public static boolean isCreate(UpdateTableRequest request) { } private UserSecretsManager getUserSecretsManager() { - // TODO: Wire into appropriate factories and/or contexts. - return UnsafeInMemorySecretsManager.GLOBAL_INSTANCE; + return userSecretsManager; } @Override diff --git a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java index 2332e9a00d..7ffd72c448 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java @@ -34,6 +34,7 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; @@ -53,15 +54,18 @@ public class PolarisCallContextCatalogFactory implements CallContextCatalogFacto private final TaskExecutor taskExecutor; private final FileIOFactory fileIOFactory; private final MetaStoreManagerFactory metaStoreManagerFactory; + private final UserSecretsManagerFactory userSecretsManagerFactory; @Inject public PolarisCallContextCatalogFactory( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, + UserSecretsManagerFactory userSecretsManagerFactory, TaskExecutor taskExecutor, FileIOFactory fileIOFactory) { this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; + this.userSecretsManagerFactory = userSecretsManagerFactory; this.taskExecutor = taskExecutor; this.fileIOFactory = fileIOFactory; } diff --git a/service/common/src/main/java/org/apache/polaris/service/secrets/UnsafeInMemorySecretsManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/secrets/UnsafeInMemorySecretsManagerFactory.java new file mode 100644 index 0000000000..fb11b9acd7 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/secrets/UnsafeInMemorySecretsManagerFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.secrets; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.secrets.UnsafeInMemorySecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; + +@ApplicationScoped +@Identifier("in-memory") +public class UnsafeInMemorySecretsManagerFactory implements UserSecretsManagerFactory { + private final Map cachedSecretsManagers = new ConcurrentHashMap<>(); + + @Override + public UserSecretsManager getOrCreateUserSecretsManager(RealmContext realmContext) { + return cachedSecretsManagers.computeIfAbsent( + realmContext.getRealmIdentifier(), key -> new UnsafeInMemorySecretsManager()); + } +} diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index c763af661c..3e8eded344 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -42,6 +42,8 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.secrets.UserSecretsManager; +import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; import org.apache.polaris.service.catalog.DefaultCatalogPrefixParser; @@ -55,6 +57,7 @@ import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.secrets.UnsafeInMemorySecretsManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.apache.polaris.service.task.TaskExecutor; import org.assertj.core.util.TriFunction; @@ -131,11 +134,15 @@ public TestServices build() { storageIntegrationProvider, polarisDiagnostics); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(metaStoreManagerFactory) {}; + UserSecretsManagerFactory userSecretsManagerFactory = + new UnsafeInMemorySecretsManagerFactory(); PolarisEntityManager entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + UserSecretsManager userSecretsManager = + userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); CallContext callContext = @@ -168,7 +175,11 @@ public Map contextVariables() { CallContextCatalogFactory callContextFactory = new PolarisCallContextCatalogFactory( - realmEntityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); + realmEntityManagerFactory, + metaStoreManagerFactory, + userSecretsManagerFactory, + taskExecutor, + fileIOFactory); IcebergRestCatalogApiService service = new IcebergCatalogAdapter( @@ -177,6 +188,7 @@ public Map contextVariables() { callContextFactory, entityManager, metaStoreManager, + userSecretsManager, authorizer, new DefaultCatalogPrefixParser()); @@ -220,7 +232,11 @@ public String getAuthenticationScheme() { PolarisCatalogsApi catalogsApi = new PolarisCatalogsApi( new PolarisServiceImpl( - realmEntityManagerFactory, metaStoreManagerFactory, authorizer, callContext)); + realmEntityManagerFactory, + metaStoreManagerFactory, + userSecretsManagerFactory, + authorizer, + callContext)); return new TestServices( catalogsApi, From cbe362d3fee7af13eb2891457876c5483c2b6818 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Fri, 4 Apr 2025 06:16:25 +0000 Subject: [PATCH 13/17] Gate all federation logic behind a new FeatureConfiguration - ENABLE_CATALOG_FEDERATION --- .../core/config/FeatureConfiguration.java | 26 +++++++++++++++++++ .../main/resources/application-it.properties | 1 + .../src/main/resources/application.properties | 2 ++ .../service/admin/PolarisAdminService.java | 7 ++++- .../iceberg/IcebergCatalogHandler.java | 2 ++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index cc8bae454d..d86f0679c0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Optional; import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.context.CallContext; /** * Configurations for features within Polaris. These configurations are intended to be customized @@ -35,6 +36,22 @@ protected FeatureConfiguration( super(key, description, defaultValue, catalogConfig); } + /** + * Helper for the common scenario of gating a feature with a boolean FeatureConfiguration, where + * we want to throw an UnsupportedOperationException if it's not enabled. + */ + public static void enforceFeatureEnabledOrThrow( + CallContext callContext, FeatureConfiguration featureConfig) { + boolean enabled = + callContext + .getPolarisCallContext() + .getConfigurationStore() + .getConfiguration(callContext.getPolarisCallContext(), featureConfig); + if (!enabled) { + throw new UnsupportedOperationException("Feature not enabled: " + featureConfig.key); + } + } + public static final FeatureConfiguration ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING = PolarisConfiguration.builder() @@ -190,4 +207,13 @@ protected FeatureConfiguration( .description("If true, the generic-tables endpoints are enabled") .defaultValue(false) .buildFeatureConfiguration(); + + public static final FeatureConfiguration ENABLE_CATALOG_FEDERATION = + PolarisConfiguration.builder() + .key("ENABLE_CATALOG_FEDERATION") + .description( + "If true, allows creating and using ExternalCatalogs containing ConnectionConfigInfos" + + " to perform federation to remote catalogs.") + .defaultValue(false) + .buildFeatureConfiguration(); } diff --git a/quarkus/defaults/src/main/resources/application-it.properties b/quarkus/defaults/src/main/resources/application-it.properties index 906d13c1ed..4419d4d590 100644 --- a/quarkus/defaults/src/main/resources/application-it.properties +++ b/quarkus/defaults/src/main/resources/application-it.properties @@ -40,6 +40,7 @@ polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKI polaris.features.defaults."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_it"=true polaris.features.defaults."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] +polaris.features.defaults."ENABLE_CATALOG_FEDERATION"=true polaris.realm-context.realms=POLARIS,OTHER diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index d6f579910b..c34c1bc0e9 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -91,6 +91,8 @@ polaris.realm-context.require-header=false polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] +# polaris.features.defaults."ENABLE_CATALOG_FEDERATION"=true + # realm overrides # polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true # polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 133249a0f6..17ea0ccc44 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -674,7 +674,12 @@ public PolarisEntity createCatalog(CreateCatalogRequest catalogRequest) { .build(); if (requiresSecretReferenceExtraction(catalogRequest)) { - // TODO: Also gate this on a feature parameter + LOGGER + .atDebug() + .addKeyValue("catalogName", entity.getName()) + .log("Extracting secret references to create federated catalog"); + FeatureConfiguration.enforceFeatureEnabledOrThrow( + callContext, FeatureConfiguration.ENABLE_CATALOG_FEDERATION); // For fields that contain references to secrets, we'll separately process the secrets from // the original request first, and then populate those fields with the extracted secret // references as part of the construction of the internal persistence entity. diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index e8f46e15ae..dc908ceed4 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -176,6 +176,8 @@ protected void initializeCatalog() { .atInfo() .addKeyValue("remoteUrl", connectionConfigurationInfo.getUri()) .log("Initializing federated catalog"); + FeatureConfiguration.enforceFeatureEnabledOrThrow( + callContext, FeatureConfiguration.ENABLE_CATALOG_FEDERATION); Catalog federatedCatalog; switch (connectionConfigurationInfo.getConnectionType()) { From 3e85244f9ff4f110f03faa52b72a5a1c234e61f0 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Fri, 4 Apr 2025 06:22:33 +0000 Subject: [PATCH 14/17] Also rename some variables and method names to be consistent with prior rename to ConnectionConfigInfoDpo --- .../polaris/core/entity/CatalogEntity.java | 6 +++--- .../connection/ConnectionConfigInfoDpoTest.java | 16 ++++++++-------- .../service/admin/PolarisAdminService.java | 2 +- .../catalog/iceberg/IcebergCatalogHandler.java | 17 ++++++++--------- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index ff4ce894a0..a995883304 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -172,7 +172,7 @@ private StorageConfigInfo getStorageInfo(Map internalProperties) private ConnectionConfigInfo getConnectionInfo(Map internalProperties) { if (internalProperties.containsKey( PolarisEntityConstants.getConnectionConfigInfoPropertyName())) { - ConnectionConfigInfoDpo configInfo = getConnectionConfigurationInfo(); + ConnectionConfigInfoDpo configInfo = getConnectionConfigInfoDpo(); return configInfo.asConnectionConfigInfoModel(); } return null; @@ -207,7 +207,7 @@ public boolean isPassthroughFacade() { .containsKey(PolarisEntityConstants.getConnectionConfigInfoPropertyName()); } - public ConnectionConfigInfoDpo getConnectionConfigurationInfo() { + public ConnectionConfigInfoDpo getConnectionConfigInfoDpo() { String configStr = getInternalPropertiesAsMap() .get(PolarisEntityConstants.getConnectionConfigInfoPropertyName()); @@ -317,7 +317,7 @@ private void validateMaxAllowedLocations(Collection allowedLocations) { } } - public Builder setConnectionConfigurationInfoWithSecrets( + public Builder setConnectionConfigInfoDpoWithSecrets( ConnectionConfigInfo connectionConfigurationModel, Map secretReferences) { if (connectionConfigurationModel != null) { diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java index b2e7ac99c6..c212e620e3 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java @@ -52,12 +52,12 @@ void testOAuthClientCredentialsParameters() throws JsonProcessingException { + " \"scopes\": [\"PRINCIPAL_ROLE:ALL\"]" + " }" + "}"; - ConnectionConfigInfoDpo connectionConfigurationInfo = + ConnectionConfigInfoDpo connectionConfigInfoDpo = ConnectionConfigInfoDpo.deserialize(polarisDiagnostics, json); - Assertions.assertNotNull(connectionConfigurationInfo); - System.out.println(connectionConfigurationInfo.serialize()); + Assertions.assertNotNull(connectionConfigInfoDpo); + System.out.println(connectionConfigInfoDpo.serialize()); JsonNode tree1 = objectMapper.readTree(json); - JsonNode tree2 = objectMapper.readTree(connectionConfigurationInfo.serialize()); + JsonNode tree2 = objectMapper.readTree(connectionConfigInfoDpo.serialize()); Assertions.assertEquals(tree1, tree2); } @@ -80,12 +80,12 @@ void testBearerAuthenticationParameters() throws JsonProcessingException { + " }" + " }" + "}"; - ConnectionConfigInfoDpo connectionConfigurationInfo = + ConnectionConfigInfoDpo connectionConfigInfoDpo = ConnectionConfigInfoDpo.deserialize(polarisDiagnostics, json); - Assertions.assertNotNull(connectionConfigurationInfo); - System.out.println(connectionConfigurationInfo.serialize()); + Assertions.assertNotNull(connectionConfigInfoDpo); + System.out.println(connectionConfigInfoDpo.serialize()); JsonNode tree1 = objectMapper.readTree(json); - JsonNode tree2 = objectMapper.readTree(connectionConfigurationInfo.serialize()); + JsonNode tree2 = objectMapper.readTree(connectionConfigInfoDpo.serialize()); Assertions.assertEquals(tree1, tree2); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 17ea0ccc44..993f731d39 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -687,7 +687,7 @@ public PolarisEntity createCatalog(CreateCatalogRequest catalogRequest) { extractSecretReferences(catalogRequest, entity); entity = new CatalogEntity.Builder(entity) - .setConnectionConfigurationInfoWithSecrets( + .setConnectionConfigInfoDpoWithSecrets( ((ExternalCatalog) catalogRequest.getCatalog()).getConnectionConfigInfo(), processedSecretReferences) .build(); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index dc908ceed4..bd282e0ddc 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -169,18 +169,18 @@ private UserSecretsManager getUserSecretsManager() { protected void initializeCatalog() { CatalogEntity resolvedCatalogEntity = CatalogEntity.of(resolutionManifest.getResolvedReferenceCatalogEntity().getRawLeafEntity()); - ConnectionConfigInfoDpo connectionConfigurationInfo = - resolvedCatalogEntity.getConnectionConfigurationInfo(); - if (connectionConfigurationInfo != null) { + ConnectionConfigInfoDpo connectionConfigInfoDpo = + resolvedCatalogEntity.getConnectionConfigInfoDpo(); + if (connectionConfigInfoDpo != null) { LOGGER .atInfo() - .addKeyValue("remoteUrl", connectionConfigurationInfo.getUri()) + .addKeyValue("remoteUrl", connectionConfigInfoDpo.getUri()) .log("Initializing federated catalog"); FeatureConfiguration.enforceFeatureEnabledOrThrow( callContext, FeatureConfiguration.ENABLE_CATALOG_FEDERATION); Catalog federatedCatalog; - switch (connectionConfigurationInfo.getConnectionType()) { + switch (connectionConfigInfoDpo.getConnectionType()) { case ICEBERG_REST: SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); federatedCatalog = @@ -191,13 +191,12 @@ protected void initializeCatalog() { .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) .build()); federatedCatalog.initialize( - ((IcebergRestConnectionConfigInfoDpo) connectionConfigurationInfo) - .getRemoteCatalogName(), - connectionConfigurationInfo.asIcebergCatalogProperties(getUserSecretsManager())); + ((IcebergRestConnectionConfigInfoDpo) connectionConfigInfoDpo).getRemoteCatalogName(), + connectionConfigInfoDpo.asIcebergCatalogProperties(getUserSecretsManager())); break; default: throw new UnsupportedOperationException( - "Connection type not supported: " + connectionConfigurationInfo.getConnectionType()); + "Connection type not supported: " + connectionConfigInfoDpo.getConnectionType()); } this.baseCatalog = federatedCatalog; } else { From 03af9f545c50c3eebc9e25987c1c7be2a55a0ba0 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Mon, 14 Apr 2025 07:41:45 +0000 Subject: [PATCH 15/17] Change ConnectionType and AuthenticationType to be stored as int codes in persistence objects. Address PR feedback for various nits and javadoc comments. --- .../AuthenticationParametersDpo.java | 31 +++++----- .../core/connection/AuthenticationType.java | 22 ++++++- .../BearerAuthenticationParametersDpo.java | 11 ++-- .../connection/ConnectionConfigInfoDpo.java | 38 ++++++++---- .../core/connection/ConnectionType.java | 59 ++++++++++++++++++- .../IcebergRestConnectionConfigInfoDpo.java | 12 ++-- .../OAuthClientCredentialsParametersDpo.java | 10 +++- .../core/secrets/UserSecretReference.java | 10 +++- .../core/secrets/UserSecretsManager.java | 6 +- .../ConnectionConfigInfoDpoTest.java | 50 ++++++++++++++-- .../iceberg/IcebergCatalogHandler.java | 7 ++- 11 files changed, 204 insertions(+), 52 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java index b9e45ae9ac..35687feeb9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationParametersDpo.java @@ -21,34 +21,36 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import jakarta.annotation.Nonnull; import java.util.Map; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; import org.apache.polaris.core.secrets.UserSecretReference; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "authenticationType", visible = true) +/** + * The internal persistence-object counterpart to AuthenticationParameters defined in the API model. + * Important: JsonSubTypes must be kept in sync with {@link AuthenticationType}. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "authenticationTypeCode", visible = true) @JsonSubTypes({ - @JsonSubTypes.Type(value = OAuthClientCredentialsParametersDpo.class, name = "OAUTH"), - @JsonSubTypes.Type(value = BearerAuthenticationParametersDpo.class, name = "BEARER"), + @JsonSubTypes.Type(value = OAuthClientCredentialsParametersDpo.class, name = "1"), + @JsonSubTypes.Type(value = BearerAuthenticationParametersDpo.class, name = "2"), }) public abstract class AuthenticationParametersDpo implements IcebergCatalogPropertiesProvider { public static final String INLINE_CLIENT_SECRET_REFERENCE_KEY = "inlineClientSecretReference"; public static final String INLINE_BEARER_TOKEN_REFERENCE_KEY = "inlineBearerTokenReference"; - @JsonProperty(value = "authenticationType") - private final AuthenticationType authenticationType; + @JsonProperty(value = "authenticationTypeCode") + private final int authenticationTypeCode; public AuthenticationParametersDpo( - @JsonProperty(value = "authenticationType", required = true) @Nonnull - AuthenticationType authenticationType) { - this.authenticationType = authenticationType; + @JsonProperty(value = "authenticationTypeCode", required = true) int authenticationTypeCode) { + this.authenticationTypeCode = authenticationTypeCode; } - public @Nonnull AuthenticationType getAuthenticationType() { - return authenticationType; + public int getAuthenticationTypeCode() { + return authenticationTypeCode; } public abstract AuthenticationParameters asAuthenticationParametersModel(); @@ -56,14 +58,14 @@ public AuthenticationParametersDpo( public static AuthenticationParametersDpo fromAuthenticationParametersModelWithSecrets( AuthenticationParameters authenticationParameters, Map secretReferences) { - AuthenticationParametersDpo config = null; + final AuthenticationParametersDpo config; switch (authenticationParameters.getAuthenticationType()) { case OAUTH: OAuthClientCredentialsParameters oauthClientCredentialsModel = (OAuthClientCredentialsParameters) authenticationParameters; config = new OAuthClientCredentialsParametersDpo( - AuthenticationType.OAUTH, + AuthenticationType.OAUTH.getCode(), oauthClientCredentialsModel.getTokenUri(), oauthClientCredentialsModel.getClientId(), secretReferences.get(INLINE_CLIENT_SECRET_REFERENCE_KEY), @@ -74,7 +76,8 @@ public static AuthenticationParametersDpo fromAuthenticationParametersModelWithS (BearerAuthenticationParameters) authenticationParameters; config = new BearerAuthenticationParametersDpo( - AuthenticationType.BEARER, secretReferences.get(INLINE_BEARER_TOKEN_REFERENCE_KEY)); + AuthenticationType.BEARER.getCode(), + secretReferences.get(INLINE_BEARER_TOKEN_REFERENCE_KEY)); break; default: throw new IllegalStateException( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java index 03608bba88..8ccc529854 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/AuthenticationType.java @@ -18,7 +18,25 @@ */ package org.apache.polaris.core.connection; +/** + * The internal persistence-object counterpart to AuthenticationParameters.AuthenticationTypeEnum + * defined in the API model. We define integer type codes in this enum for better compatibility + * within persisted data in case the names of enum types are ever changed in place. + * + *

Important: Codes must be kept in-sync with JsonSubTypes annotated within {@link + * AuthenticationParametersDpo}. + */ public enum AuthenticationType { - OAUTH, - BEARER + OAUTH(1), + BEARER(2); + + private final int code; + + AuthenticationType(int code) { + this.code = code; + } + + public int getCode() { + return this.code; + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java index 3499e4cb30..b5d136bc5d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/BearerAuthenticationParametersDpo.java @@ -28,17 +28,20 @@ import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.secrets.UserSecretsManager; +/** + * The internal persistence-object counterpart to BearerAuthenticationParameters defined in the API + * model. + */ public class BearerAuthenticationParametersDpo extends AuthenticationParametersDpo { @JsonProperty(value = "bearerTokenReference") private final UserSecretReference bearerTokenReference; public BearerAuthenticationParametersDpo( - @JsonProperty(value = "authenticationType", required = true) @Nonnull - AuthenticationType authenticationType, + @JsonProperty(value = "authenticationTypeCode", required = true) int authenticationTypeCode, @JsonProperty(value = "bearerTokenReference", required = true) @Nonnull UserSecretReference bearerTokenReference) { - super(authenticationType); + super(authenticationTypeCode); this.bearerTokenReference = bearerTokenReference; } @@ -63,7 +66,7 @@ public AuthenticationParameters asAuthenticationParametersModel() { @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("authenticationType", getAuthenticationType()) + .add("authenticationTypeCode", getAuthenticationTypeCode()) .add("bearerTokenReference", getBearerTokenReference()) .toString(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java index d4bd48942d..c5bf56f292 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpo.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import java.net.MalformedURLException; import java.net.URI; import java.net.URL; @@ -37,15 +38,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "connectionType", visible = true) +/** + * The internal persistence-object counterpart to ConnectionConfigInfo defined in the API model. + * Important: JsonSubTypes must be kept in sync with {@link ConnectionType}. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "connectionTypeCode", visible = true) @JsonSubTypes({ - @JsonSubTypes.Type(value = IcebergRestConnectionConfigInfoDpo.class, name = "ICEBERG_REST"), + @JsonSubTypes.Type(value = IcebergRestConnectionConfigInfoDpo.class, name = "1"), }) public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertiesProvider { private static final Logger logger = LoggerFactory.getLogger(ConnectionConfigInfoDpo.class); // The type of the connection - private final ConnectionType connectionType; + private final int connectionTypeCode; // The URI of the remote catalog private final String uri; @@ -54,20 +59,19 @@ public abstract class ConnectionConfigInfoDpo implements IcebergCatalogPropertie private final AuthenticationParametersDpo authenticationParameters; public ConnectionConfigInfoDpo( - @JsonProperty(value = "connectionType", required = true) @Nonnull - ConnectionType connectionType, + @JsonProperty(value = "connectionTypeCode", required = true) int connectionTypeCode, @JsonProperty(value = "uri", required = true) @Nonnull String uri, @JsonProperty(value = "authenticationParameters", required = true) @Nonnull AuthenticationParametersDpo authenticationParameters) { - this(connectionType, uri, authenticationParameters, true); + this(connectionTypeCode, uri, authenticationParameters, true); } protected ConnectionConfigInfoDpo( - @Nonnull ConnectionType connectionType, + int connectionTypeCode, @Nonnull String uri, @Nonnull AuthenticationParametersDpo authenticationParameters, boolean validateUri) { - this.connectionType = connectionType; + this.connectionTypeCode = connectionTypeCode; this.uri = uri; this.authenticationParameters = authenticationParameters; if (validateUri) { @@ -75,8 +79,8 @@ protected ConnectionConfigInfoDpo( } } - public ConnectionType getConnectionType() { - return connectionType; + public int getConnectionTypeCode() { + return connectionTypeCode; } public String getUri() { @@ -103,7 +107,7 @@ public String serialize() { } } - public static ConnectionConfigInfoDpo deserialize( + public static @Nullable ConnectionConfigInfoDpo deserialize( @Nonnull PolarisDiagnostics diagnostics, final @Nonnull String jsonStr) { try { return DEFAULT_MAPPER.readValue(jsonStr, ConnectionConfigInfoDpo.class); @@ -124,6 +128,11 @@ protected void validateUri(String uri) { } } + /** + * Converts from the API-model ConnectionConfigInfo by merging basic carryover fields with + * expected associated secretReference(s) that have been previously scrubbed or resolved from + * inline secret fields of the API request. + */ public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( ConnectionConfigInfo connectionConfigurationModel, Map secretReferences) { @@ -137,7 +146,7 @@ public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( icebergRestConfigModel.getAuthenticationParameters(), secretReferences); config = new IcebergRestConnectionConfigInfoDpo( - ConnectionType.ICEBERG_REST, + ConnectionType.ICEBERG_REST.getCode(), icebergRestConfigModel.getUri(), authenticationParameters, icebergRestConfigModel.getRemoteCatalogName()); @@ -149,5 +158,10 @@ public static ConnectionConfigInfoDpo fromConnectionConfigInfoModelWithSecrets( return config; } + /** + * Produces the correponding API-model ConnectionConfigInfo for this persistence object; many + * fields are one-to-one direct mappings, but some fields, such as secretReferences, might only be + * applicable/present in the persistence object, but not the API model object. + */ public abstract ConnectionConfigInfo asConnectionConfigInfoModel(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java index 5be343d180..ef49cb5cab 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/ConnectionType.java @@ -18,6 +18,63 @@ */ package org.apache.polaris.core.connection; +import jakarta.annotation.Nullable; + +/** + * The internal persistence-object counterpart to ConnectionConfigInfo.ConnectionTypeEnum defined in + * the API model. We define integer type codes in this enum for better compatibility within + * persisted data in case the names of enum types are ever changed in place. + * + *

Important: Codes must be kept in-sync with JsonSubTypes annotated within {@link + * ConnectionConfigInfoDpo}. + */ public enum ConnectionType { - ICEBERG_REST + ICEBERG_REST(1); + + private static final ConnectionType[] REVERSE_MAPPING_ARRAY; + + static { + // find max array size + int maxId = 0; + for (ConnectionType connectionType : ConnectionType.values()) { + if (maxId < connectionType.code) { + maxId = connectionType.code; + } + } + + // allocate mapping array + REVERSE_MAPPING_ARRAY = new ConnectionType[maxId + 1]; + + // populate mapping array + for (ConnectionType connectionType : ConnectionType.values()) { + REVERSE_MAPPING_ARRAY[connectionType.code] = connectionType; + } + } + + private final int code; + + ConnectionType(int code) { + this.code = code; + } + + /** + * Given the code associated to the type, return the associated ConnectionType. Return null if not + * found + * + * @param connectionTypeCode code associated to the entity type + * @return ConnectionType corresponding to that code or null if mapping not found + */ + public static @Nullable ConnectionType fromCode(int connectionTypeCode) { + // ensure it is within bounds + if (connectionTypeCode >= REVERSE_MAPPING_ARRAY.length) { + return null; + } + + // get value + return REVERSE_MAPPING_ARRAY[connectionTypeCode]; + } + + public int getCode() { + return this.code; + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java index b138370726..2ebe099519 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergRestConnectionConfigInfoDpo.java @@ -29,20 +29,23 @@ import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; import org.apache.polaris.core.secrets.UserSecretsManager; +/** + * The internal persistence-object counterpart to IcebergRestConnectionConfigInfo defined in the API + * model. + */ public class IcebergRestConnectionConfigInfoDpo extends ConnectionConfigInfoDpo implements IcebergCatalogPropertiesProvider { private final String remoteCatalogName; public IcebergRestConnectionConfigInfoDpo( - @JsonProperty(value = "connectionType", required = true) @Nonnull - ConnectionType connectionType, + @JsonProperty(value = "connectionTypeCode", required = true) int connectionTypeCode, @JsonProperty(value = "uri", required = true) @Nonnull String uri, @JsonProperty(value = "authenticationParameters", required = true) @Nonnull AuthenticationParametersDpo authenticationParameters, @JsonProperty(value = "remoteCatalogName", required = false) @Nullable String remoteCatalogName) { - super(connectionType, uri, authenticationParameters); + super(connectionTypeCode, uri, authenticationParameters); this.remoteCatalogName = remoteCatalogName; } @@ -76,8 +79,7 @@ public ConnectionConfigInfo asConnectionConfigInfoModel() { @Override public String toString() { return MoreObjects.toStringHelper(this) - .add("connectionType", getConnectionType()) - .add("connectionType", getConnectionType().name()) + .add("connectionTypeCode", getConnectionTypeCode()) .add("uri", getUri()) .add("remoteCatalogName", getRemoteCatalogName()) .add("authenticationParameters", getAuthenticationParameters().toString()) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java index e80a8467d3..127ed3e3c4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/OAuthClientCredentialsParametersDpo.java @@ -38,6 +38,10 @@ import org.apache.polaris.core.secrets.UserSecretReference; import org.apache.polaris.core.secrets.UserSecretsManager; +/** + * The internal persistence-object counterpart to OAuthClientCredentialsParameters defined in the + * API model. + */ public class OAuthClientCredentialsParametersDpo extends AuthenticationParametersDpo { private static final Joiner COLON_JOINER = Joiner.on(":"); @@ -55,14 +59,13 @@ public class OAuthClientCredentialsParametersDpo extends AuthenticationParameter private final List scopes; public OAuthClientCredentialsParametersDpo( - @JsonProperty(value = "authenticationType", required = true) @Nonnull - AuthenticationType authenticationType, + @JsonProperty(value = "authenticationTypeCode", required = true) int authenticationTypeCode, @JsonProperty(value = "tokenUri", required = false) @Nullable String tokenUri, @JsonProperty(value = "clientId", required = true) @Nonnull String clientId, @JsonProperty(value = "clientSecretReference", required = true) @Nonnull UserSecretReference clientSecretReference, @JsonProperty(value = "scopes", required = false) @Nullable List scopes) { - super(authenticationType); + super(authenticationTypeCode); this.tokenUri = tokenUri; this.clientId = clientId; @@ -139,6 +142,7 @@ protected void validateTokenUri(String tokenUri) { @Override public String toString() { return MoreObjects.toStringHelper(this) + .add("authenticationTypeCode", getAuthenticationTypeCode()) .add("tokenUri", getTokenUri()) .add("clientId", getClientId()) .add("clientSecretReference", getClientSecretReference()) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java index f4faeb5062..c769269d39 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java @@ -54,6 +54,14 @@ public class UserSecretReference { @JsonProperty(value = "referencePayload") private final Map referencePayload; + /** + * @param urn A string which should be self-sufficient to retrieve whatever secret material that + * is stored in the remote secret store. + * @param referencePayload Optionally, any additional information that is necessary to fully + * reconstitute the original secret based on what is retrieved by the {@code urn}; this + * payload may include hashes/checksums, encryption key ids, OTP encryption keys, additional + * protocol/version specifiers, etc., which are implementation-specific. + */ public UserSecretReference( @JsonProperty(value = "urn", required = true) @Nonnull String urn, @JsonProperty(value = "referencePayload") @Nullable Map referencePayload) { @@ -71,7 +79,7 @@ public UserSecretReference( @Override public int hashCode() { - return Objects.hashCode(getUrn()) ^ Objects.hashCode(getReferencePayload()); + return Objects.hash(getUrn(), getReferencePayload()); } @Override diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java index 8df656e4ee..7eb699f2c6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java @@ -47,7 +47,8 @@ public interface UserSecretsManager { UserSecretReference writeSecret(@Nonnull String secret, @Nonnull PolarisEntity forEntity); /** - * Retrieve a secret using the {@code secretReference}. + * Retrieve a secret using the {@code secretReference}. See {@link UserSecretReference} for + * details about identifiers and payloads. * * @param secretReference Identifier and any associated payload used for retrieving the secret * @return The stored secret, or null if it no longer exists @@ -56,7 +57,8 @@ public interface UserSecretsManager { String readSecret(@Nonnull UserSecretReference secretReference); /** - * Delete a stored secret + * Delete a stored secret. See {@link UserSecretReference} for details about identifiers and + * payloads. * * @param secretReference Identifier and any associated payload used for retrieving the secret */ diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java index c212e620e3..515ac68999 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -32,14 +33,15 @@ public class ConnectionConfigInfoDpoTest { @Test void testOAuthClientCredentialsParameters() throws JsonProcessingException { + // Test deserialization and reserialization of the persistence JSON. String json = "" + "{" - + " \"connectionType\": \"ICEBERG_REST\"," + + " \"connectionTypeCode\": 1," + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + " \"remoteCatalogName\": \"my-catalog\"," + " \"authenticationParameters\": {" - + " \"authenticationType\": \"OAUTH\"," + + " \"authenticationTypeCode\": 1," + " \"tokenUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog/v1/oauth/tokens\"," + " \"clientId\": \"client-id\"," + " \"clientSecretReference\": {" @@ -55,22 +57,42 @@ void testOAuthClientCredentialsParameters() throws JsonProcessingException { ConnectionConfigInfoDpo connectionConfigInfoDpo = ConnectionConfigInfoDpo.deserialize(polarisDiagnostics, json); Assertions.assertNotNull(connectionConfigInfoDpo); - System.out.println(connectionConfigInfoDpo.serialize()); JsonNode tree1 = objectMapper.readTree(json); JsonNode tree2 = objectMapper.readTree(connectionConfigInfoDpo.serialize()); Assertions.assertEquals(tree1, tree2); + + // Test conversion into API model JSON. + ConnectionConfigInfo connectionConfigInfoApiModel = + connectionConfigInfoDpo.asConnectionConfigInfoModel(); + String expectedApiModelJson = + "" + + "{" + + " \"connectionType\": \"ICEBERG_REST\"," + + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"remoteCatalogName\": \"my-catalog\"," + + " \"authenticationParameters\": {" + + " \"authenticationType\": \"OAUTH\"," + + " \"tokenUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog/v1/oauth/tokens\"," + + " \"clientId\": \"client-id\"," + + " \"scopes\": [\"PRINCIPAL_ROLE:ALL\"]" + + " }" + + "}"; + Assertions.assertEquals( + objectMapper.readValue(expectedApiModelJson, ConnectionConfigInfo.class), + connectionConfigInfoApiModel); } @Test void testBearerAuthenticationParameters() throws JsonProcessingException { + // Test deserialization and reserialization of the persistence JSON. String json = "" + "{" - + " \"connectionType\": \"ICEBERG_REST\"," + + " \"connectionTypeCode\": 1," + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + " \"remoteCatalogName\": \"my-catalog\"," + " \"authenticationParameters\": {" - + " \"authenticationType\": \"BEARER\"," + + " \"authenticationTypeCode\": 2," + " \"bearerTokenReference\": {" + " \"urn\": \"urn:polaris-secret:keystore-id-12345\"," + " \"referencePayload\": {" @@ -83,9 +105,25 @@ void testBearerAuthenticationParameters() throws JsonProcessingException { ConnectionConfigInfoDpo connectionConfigInfoDpo = ConnectionConfigInfoDpo.deserialize(polarisDiagnostics, json); Assertions.assertNotNull(connectionConfigInfoDpo); - System.out.println(connectionConfigInfoDpo.serialize()); JsonNode tree1 = objectMapper.readTree(json); JsonNode tree2 = objectMapper.readTree(connectionConfigInfoDpo.serialize()); Assertions.assertEquals(tree1, tree2); + + // Test conversion into API model JSON. + ConnectionConfigInfo connectionConfigInfoApiModel = + connectionConfigInfoDpo.asConnectionConfigInfoModel(); + String expectedApiModelJson = + "" + + "{" + + " \"connectionType\": \"ICEBERG_REST\"," + + " \"uri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog\"," + + " \"remoteCatalogName\": \"my-catalog\"," + + " \"authenticationParameters\": {" + + " \"authenticationType\": \"BEARER\"" + + " }" + + "}"; + Assertions.assertEquals( + objectMapper.readValue(expectedApiModelJson, ConnectionConfigInfo.class), + connectionConfigInfoApiModel); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 75e3c63f2c..716d68fdbc 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -76,6 +76,7 @@ import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.connection.ConnectionConfigInfoDpo; +import org.apache.polaris.core.connection.ConnectionType; import org.apache.polaris.core.connection.IcebergRestConnectionConfigInfoDpo; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; @@ -181,7 +182,9 @@ protected void initializeCatalog() { callContext, FeatureConfiguration.ENABLE_CATALOG_FEDERATION); Catalog federatedCatalog; - switch (connectionConfigInfoDpo.getConnectionType()) { + ConnectionType connectionType = + ConnectionType.fromCode(connectionConfigInfoDpo.getConnectionTypeCode()); + switch (connectionType) { case ICEBERG_REST: SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); federatedCatalog = @@ -197,7 +200,7 @@ protected void initializeCatalog() { break; default: throw new UnsupportedOperationException( - "Connection type not supported: " + connectionConfigInfoDpo.getConnectionType()); + "Connection type not supported: " + connectionType); } this.baseCatalog = federatedCatalog; } else { From 544ce82da8dc6f4534592f1ebeb6725dee0eb8be Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Tue, 15 Apr 2025 06:03:22 +0000 Subject: [PATCH 16/17] Add javadoc comment to IcebergCatalogPropertiesProvider --- .../core/connection/IcebergCatalogPropertiesProvider.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java index df6897854e..e7955bc61a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/connection/IcebergCatalogPropertiesProvider.java @@ -22,6 +22,12 @@ import java.util.Map; import org.apache.polaris.core.secrets.UserSecretsManager; +/** + * Configuration wrappers which ultimately translate their contents into Iceberg properties and + * which may hold other nested configuration wrapper objects implement this interface to allow + * delegating type-specific configuration translation logic to subclasses instead of needing to + * expose the internals of deeply nested configuration objects to a visitor class. + */ public interface IcebergCatalogPropertiesProvider { @Nonnull Map asIcebergCatalogProperties(UserSecretsManager secretsManager); From beaa34ebdf4f8c349cdba254b0f35f00cd68315e Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Thu, 17 Apr 2025 22:24:24 +0000 Subject: [PATCH 17/17] Add some constraints on the expected format of the URN in UserSecretReference and placeholders for next steps where we'd provide a ResolvingUserSecretsManager for example if the runtime ever needs to delegate to two different implementations of UserSecretsManager for different entities. Reduce the `forEntity` argument to just PolarisEntityCore to make it more clear that the implementation is supposed to extract the necessary identifier info from forEntity for backend cleanup and tracking purposes. --- .../secrets/UnsafeInMemorySecretsManager.java | 5 ++-- .../core/secrets/UserSecretReference.java | 25 ++++++++++++++++++- .../core/secrets/UserSecretsManager.java | 18 ++++++------- .../ConnectionConfigInfoDpoTest.java | 4 +-- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java index fcf255ee31..22e8c53786 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UnsafeInMemorySecretsManager.java @@ -27,7 +27,7 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityCore; /** * A minimal in-memory implementation of UserSecretsManager that should only be used for test and @@ -48,7 +48,8 @@ public class UnsafeInMemorySecretsManager implements UserSecretsManager { /** {@inheritDoc} */ @Override @Nonnull - public UserSecretReference writeSecret(@Nonnull String secret, @Nonnull PolarisEntity forEntity) { + public UserSecretReference writeSecret( + @Nonnull String secret, @Nonnull PolarisEntityCore forEntity) { // For illustrative purposes and to exercise the control flow of requiring both the stored // secret as well as the secretReferencePayload to recover the original secret, we'll use // basic XOR encryption and store the randomly generated key in the reference payload. diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java index c769269d39..7181acb041 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretReference.java @@ -18,8 +18,10 @@ */ package org.apache.polaris.core.secrets; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.util.HashMap; @@ -56,7 +58,10 @@ public class UserSecretReference { /** * @param urn A string which should be self-sufficient to retrieve whatever secret material that - * is stored in the remote secret store. + * is stored in the remote secret store and also to identify an implementation of the + * UserSecretsManager which is capable of interpreting this concrete UserSecretReference. + * Should be of the form: + * 'urn:polaris-secret:<secret-manager-type>:<type-specific-identifier> * @param referencePayload Optionally, any additional information that is necessary to fully * reconstitute the original secret based on what is retrieved by the {@code urn}; this * payload may include hashes/checksums, encryption key ids, OTP encryption keys, additional @@ -65,10 +70,28 @@ public class UserSecretReference { public UserSecretReference( @JsonProperty(value = "urn", required = true) @Nonnull String urn, @JsonProperty(value = "referencePayload") @Nullable Map referencePayload) { + // TODO: Add better/standardized parsing and validation of URN syntax + Preconditions.checkArgument( + urn.startsWith("urn:polaris-secret:") && urn.split(":").length >= 4, + "Invalid secret URN '%s'; must be of the form " + + "'urn:polaris-secret::'", + urn); this.urn = urn; this.referencePayload = Objects.requireNonNullElse(referencePayload, new HashMap<>()); } + /** + * Since UserSecretReference objects are specific to UserSecretManager implementations, the + * "secret-manager-type" portion of the URN should be used to validate that a URN is valid for a + * given implementation and to dispatch to the correct implementation at runtime if multiple + * concurrent implementations are possible in a given runtime environment. + */ + @JsonIgnore + public String getUserSecretManagerTypeFromUrn() { + // TODO: Add better/standardized parsing and validation of URN syntax + return urn.split(":")[2]; + } + public @Nonnull String getUrn() { return urn; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java index 7eb699f2c6..b1418efc9a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/secrets/UserSecretsManager.java @@ -19,7 +19,7 @@ package org.apache.polaris.core.secrets; import jakarta.annotation.Nonnull; -import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityCore; /** * Manages secrets specified by users of the Polaris API, either directly or as an intermediary @@ -32,11 +32,11 @@ public interface UserSecretsManager { /** * Persist the {@code secret} under a new URN {@code secretUrn} and return a {@code * UserSecretReference} that can subsequently be used by this same UserSecretsManager to retrieve - * the original secret. The {@code forEntity} is provided for an implementation to optionally - * extract other identifying metadata such as entity type, name, etc., to store alongside the - * remotely stored secret to facilitate operational management of the secrets outside of the core - * Polaris service (for example, to perform garbage-collection if the Polaris service fails to - * delete managed secrets in the external system when associated entities are deleted. + * the original secret. The {@code forEntity} is provided for an implementation to extract other + * identifying metadata such as entity type, id, name, etc., to store alongside the remotely + * stored secret to facilitate operational management of the secrets outside of the core Polaris + * service (for example, to perform garbage-collection if the Polaris service fails to delete + * managed secrets in the external system when associated entities are deleted). * * @param secret The secret to store * @param forEntity The PolarisEntity that is associated with the secret @@ -44,13 +44,13 @@ public interface UserSecretsManager { * its entirety within a persisted PolarisEntity */ @Nonnull - UserSecretReference writeSecret(@Nonnull String secret, @Nonnull PolarisEntity forEntity); + UserSecretReference writeSecret(@Nonnull String secret, @Nonnull PolarisEntityCore forEntity); /** * Retrieve a secret using the {@code secretReference}. See {@link UserSecretReference} for * details about identifiers and payloads. * - * @param secretReference Identifier and any associated payload used for retrieving the secret + * @param secretReference Reference object for retrieving the original secret * @return The stored secret, or null if it no longer exists */ @Nonnull @@ -60,7 +60,7 @@ public interface UserSecretsManager { * Delete a stored secret. See {@link UserSecretReference} for details about identifiers and * payloads. * - * @param secretReference Identifier and any associated payload used for retrieving the secret + * @param secretReference Reference object for retrieving the original secret */ void deleteSecret(@Nonnull UserSecretReference secretReference); } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java index 515ac68999..7be1e6ed29 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/connection/ConnectionConfigInfoDpoTest.java @@ -45,7 +45,7 @@ void testOAuthClientCredentialsParameters() throws JsonProcessingException { + " \"tokenUri\": \"https://myorg-my_account.snowflakecomputing.com/polaris/api/catalog/v1/oauth/tokens\"," + " \"clientId\": \"client-id\"," + " \"clientSecretReference\": {" - + " \"urn\": \"urn:polaris-secret:keystore-id-12345\"," + + " \"urn\": \"urn:polaris-secret:secretmanager-impl:keystore-id-12345\"," + " \"referencePayload\": {" + " \"hash\": \"a1b2c3\"," + " \"encryption-key\": \"z0y9x8\"" @@ -94,7 +94,7 @@ void testBearerAuthenticationParameters() throws JsonProcessingException { + " \"authenticationParameters\": {" + " \"authenticationTypeCode\": 2," + " \"bearerTokenReference\": {" - + " \"urn\": \"urn:polaris-secret:keystore-id-12345\"," + + " \"urn\": \"urn:polaris-secret:secretmanager-impl:keystore-id-12345\"," + " \"referencePayload\": {" + " \"hash\": \"a1b2c3\"," + " \"encryption-key\": \"z0y9x8\""