From 6a896c1c162dd743dc831e1674905b712a45a6ef Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Sun, 15 Jun 2025 23:36:59 -0700 Subject: [PATCH 1/2] Add EntityTransformationEngine --- ...pseLinkPolarisMetaStoreManagerFactory.java | 10 ++- ...olarisEclipseLinkMetaStoreManagerTest.java | 2 + .../jdbc/JdbcMetaStoreManagerFactory.java | 11 ++- ...anagerWithJdbcBasePersistenceImplTest.java | 2 + .../polaris/core/PolarisCallContext.java | 20 ++++- .../EntityTransformationEngine.java | 44 +++++++++++ .../transformation/EntityTransformer.java | 41 ++++++++++ .../NoOpEntityTransformationEngine.java | 37 +++++++++ .../transformation/TransformationPoint.java | 44 +++++++++++ .../LocalPolarisMetaStoreManagerFactory.java | 15 +++- .../TransactionalMetaStoreManagerImpl.java | 10 +++ ...apAtomicOperationMetaStoreManagerTest.java | 2 + .../PolarisTreeMapMetaStoreManagerTest.java | 2 + .../core/persistence/ResolverTest.java | 7 +- .../cache/InMemoryEntityCacheTest.java | 10 ++- .../storage/BaseStorageIntegrationTest.java | 6 +- .../InMemoryStorageIntegrationTest.java | 2 + .../cache/StorageCredentialCacheTest.java | 7 +- .../admintool/config/QuarkusProducers.java | 7 ++ .../src/main/resources/application.properties | 3 + .../quarkus/config/QuarkusProducers.java | 9 ++- ...rkusEntityTransformationConfiguration.java | 44 +++++++++++ .../quarkus/admin/ManagementServiceTest.java | 4 +- .../quarkus/admin/PolarisAuthzTestBase.java | 5 ++ .../quarkus/auth/JWTRSAKeyPairTest.java | 2 +- .../auth/JWTSymmetricKeyGeneratorTest.java | 3 +- .../quarkus/catalog/IcebergCatalogTest.java | 3 + .../catalog/IcebergCatalogViewTest.java | 3 + .../PolarisGenericTableCatalogTest.java | 3 + .../quarkus/catalog/PolicyCatalogTest.java | 3 + .../config/DefaultConfigurationStoreTest.java | 2 + .../quarkus/entity/CatalogEntityTest.java | 6 +- .../task/BatchFileCleanupTaskHandlerTest.java | 11 ++- .../ManifestFileCleanupTaskHandlerTest.java | 14 +++- .../task/TableCleanupTaskHandlerTest.java | 3 + .../test/PolarisIntegrationTestFixture.java | 1 + .../test/PolarisIntegrationTestHelper.java | 2 + .../context/DefaultCallContextResolver.java | 9 ++- .../entity/transformation/AppliesTo.java | 68 ++++++++++++++++ .../entity/transformation/AppliesTos.java | 39 +++++++++ .../EntityTransformationConfiguration.java | 27 +++++++ .../EntityTransformationEngineImpl.java | 79 +++++++++++++++++++ .../transformation/NoOpEntityTransformer.java | 39 +++++++++ ...tomicOperationMetaStoreManagerFactory.java | 8 +- ...nMemoryPolarisMetaStoreManagerFactory.java | 8 +- .../service/catalog/io/FileIOFactoryTest.java | 1 + .../service/task/TaskExecutorImplTest.java | 6 +- .../apache/polaris/service/TestServices.java | 11 ++- 48 files changed, 658 insertions(+), 37 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/EntityTransformationEngine.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/EntityTransformer.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/NoOpEntityTransformationEngine.java create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/TransformationPoint.java create mode 100644 quarkus/service/src/main/java/org/apache/polaris/service/quarkus/entity/transformation/QuarkusEntityTransformationConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/entity/transformation/AppliesTo.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/entity/transformation/AppliesTos.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/entity/transformation/EntityTransformationConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/entity/transformation/EntityTransformationEngineImpl.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/entity/transformation/NoOpEntityTransformer.java diff --git a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index 9f7af1a268..4de6496b90 100644 --- a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -27,6 +27,7 @@ import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; @@ -45,15 +46,18 @@ public class EclipseLinkPolarisMetaStoreManagerFactory @Inject EclipseLinkConfiguration eclipseLinkConfiguration; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject EntityTransformationEngine entityTransformationEngine; protected EclipseLinkPolarisMetaStoreManagerFactory() { - this(null, null); + this(null, null, null); } @Inject protected EclipseLinkPolarisMetaStoreManagerFactory( - PolarisDiagnostics diagnostics, PolarisConfigurationStore configurationStore) { - super(diagnostics, configurationStore); + PolarisDiagnostics diagnostics, + PolarisConfigurationStore configurationStore, + EntityTransformationEngine entityTransformationEngine) { + super(diagnostics, configurationStore, entityTransformationEngine); } @Override diff --git a/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java b/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java index 55607981e9..3478512a02 100644 --- a/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java +++ b/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java @@ -39,6 +39,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; import org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl; @@ -96,6 +97,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { session, diagServices, new PolarisConfigurationStore() {}, + new NoOpEntityTransformationEngine(), timeSource.withZone(ZoneId.systemDefault()))); } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 118fcadd4f..8ac9f8f975 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -38,6 +38,7 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; @@ -75,6 +76,7 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { @Inject Instance dataSource; @Inject RelationalJdbcConfiguration relationalJdbcConfiguration; @Inject PolarisConfigurationStore configurationStore; + @Inject EntityTransformationEngine entityTransformationEngine; protected JdbcMetaStoreManagerFactory() {} @@ -156,7 +158,8 @@ public Map purgeRealms(Iterable realms) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); BasePersistence session = getOrCreateSessionSupplier(realmContext).get(); - PolarisCallContext callContext = new PolarisCallContext(realmContext, session, diagServices); + PolarisCallContext callContext = + new PolarisCallContext(realmContext, session, diagServices, entityTransformationEngine); BaseResult result = metaStoreManager.purge(callContext); results.put(realm, result); @@ -229,7 +232,8 @@ private PrincipalSecretsResult bootstrapServiceAndCreatePolarisPrincipalForRealm new PolarisCallContext( realmContext, sessionSupplierMap.get(realmContext.getRealmIdentifier()).get(), - diagServices); + diagServices, + entityTransformationEngine); if (CallContext.getCurrentContext() == null) { CallContext.setCurrentContext(polarisContext); } @@ -280,7 +284,8 @@ private void checkPolarisServiceBootstrappedForRealm( new PolarisCallContext( realmContext, sessionSupplierMap.get(realmContext.getRealmIdentifier()).get(), - diagServices); + diagServices, + entityTransformationEngine); if (CallContext.getCurrentContext() == null) { CallContext.setCurrentContext(polarisContext); } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java index fbcd3a958b..21e015a893 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/AtomicMetastoreManagerWithJdbcBasePersistenceImplTest.java @@ -29,6 +29,7 @@ import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; @@ -72,6 +73,7 @@ protected PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { basePersistence, diagServices, new PolarisConfigurationStore() {}, + new NoOpEntityTransformationEngine(), timeSource.withZone(ZoneId.systemDefault()))); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java index 9e5a7a8b42..9d9fba0ae9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java @@ -24,6 +24,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.BasePersistence; /** @@ -40,6 +41,8 @@ public class PolarisCallContext implements CallContext { private final PolarisConfigurationStore configurationStore; + private final EntityTransformationEngine entityTransformationEngine; + private final Clock clock; // will make it final once we remove deprecated constructor @@ -50,22 +53,26 @@ public PolarisCallContext( @Nonnull BasePersistence metaStore, @Nonnull PolarisDiagnostics diagServices, @Nonnull PolarisConfigurationStore configurationStore, + @Nonnull EntityTransformationEngine entityTransformationEngine, @Nonnull Clock clock) { this.realmContext = realmContext; this.metaStore = metaStore; this.diagServices = diagServices; this.configurationStore = configurationStore; + this.entityTransformationEngine = entityTransformationEngine; this.clock = clock; } public PolarisCallContext( @Nonnull RealmContext realmContext, @Nonnull BasePersistence metaStore, - @Nonnull PolarisDiagnostics diagServices) { + @Nonnull PolarisDiagnostics diagServices, + @Nonnull EntityTransformationEngine entityTransformationEngine) { this.realmContext = realmContext; this.metaStore = metaStore; this.diagServices = diagServices; this.configurationStore = new PolarisConfigurationStore() {}; + this.entityTransformationEngine = entityTransformationEngine; this.clock = Clock.system(ZoneId.systemDefault()); } @@ -81,6 +88,10 @@ public PolarisConfigurationStore getConfigurationStore() { return configurationStore; } + public EntityTransformationEngine getEntityTransformationEngine() { + return entityTransformationEngine; + } + public Clock getClock() { return clock; } @@ -105,6 +116,11 @@ public PolarisCallContext copy() { String realmId = this.realmContext.getRealmIdentifier(); RealmContext realmContext = () -> realmId; return new PolarisCallContext( - realmContext, this.metaStore, this.diagServices, this.configurationStore, this.clock); + realmContext, + this.metaStore, + this.diagServices, + this.configurationStore, + this.entityTransformationEngine, + this.clock); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/EntityTransformationEngine.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/EntityTransformationEngine.java new file mode 100644 index 0000000000..5c396509ba --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/EntityTransformationEngine.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.entity.transformation; + +import org.apache.polaris.core.entity.PolarisBaseEntity; + +/** + * Engine responsible for applying a sequence of {@link EntityTransformer} transformations to a + * {@link PolarisBaseEntity}. + * + *

This abstraction allows Polaris to customize or enrich entities during runtime or persistence, + * based on configured or contextual logic (e.g., injecting service identity info, computing derived + * fields). + */ +public interface EntityTransformationEngine { + /** + * Applies all registered entity transformers to the provided entity, in order. + * + * @param transformationPoint The point in the entity lifecycle where transformers should be + * applied. + * @param entity The original Polaris entity to mutate. + * @return A new transformed copy of the entity of {@link PolarisBaseEntity} after all + * transformers are applied. + */ + PolarisBaseEntity applyTransformers( + TransformationPoint transformationPoint, PolarisBaseEntity entity); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/EntityTransformer.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/EntityTransformer.java new file mode 100644 index 0000000000..f07cc61858 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/EntityTransformer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.entity.transformation; + +import org.apache.polaris.core.entity.PolarisBaseEntity; + +/** + * A transformation hook that transforms a Polaris entity. The transformer must create a new copy of + * the entity rather than updating them in-place. + * + *

Implementations of this interface apply custom logic to modify or enrich a {@link + * PolarisBaseEntity}. + */ +public interface EntityTransformer { + + /** + * Applies the transformation logic to the given entity. It can be also used to add custom logic + * around the transformation point. + * + * @param entity the entity to be transformed + * @return the transformed entity + */ + PolarisBaseEntity apply(PolarisBaseEntity entity); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/NoOpEntityTransformationEngine.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/NoOpEntityTransformationEngine.java new file mode 100644 index 0000000000..ac22c0c9ac --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/NoOpEntityTransformationEngine.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.entity.transformation; + +import org.apache.polaris.core.entity.PolarisBaseEntity; + +/** + * A no-op implementation of {@link EntityTransformationEngine} that returns the input entity + * unchanged. + * + *

This can be used in environments where entity transformation is disabled or unnecessary. + */ +public class NoOpEntityTransformationEngine implements EntityTransformationEngine { + + @Override + public PolarisBaseEntity applyTransformers( + TransformationPoint transformationPoint, final PolarisBaseEntity entity) { + return entity; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/TransformationPoint.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/TransformationPoint.java new file mode 100644 index 0000000000..fdcde0005d --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/transformation/TransformationPoint.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.core.entity.transformation; + +/** + * Defines points in the entity lifecycle where {@link EntityTransformer} can be applied. + * + *

Each transformation point corresponds to a specific hook where transformers may be executed. + * Transformers can declare which points they support, allowing the engine to invoke only the + * relevant ones. + */ +public enum TransformationPoint { + + /** Applied before a catalog entity is persisted. */ + CATALOG_PRE_PERSIST(0), + ; + + private final int id; + + TransformationPoint(int id) { + this.id = id; + } + + public int id() { + return id; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 747991636a..ea1235797d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -33,6 +33,7 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.InMemoryEntityCache; @@ -65,13 +66,16 @@ public abstract class LocalPolarisMetaStoreManagerFactory private final PolarisDiagnostics diagnostics; private final PolarisConfigurationStore configurationStore; + private final EntityTransformationEngine entityTransformationEngine; private boolean bootstrap; protected LocalPolarisMetaStoreManagerFactory( @Nonnull PolarisDiagnostics diagnostics, - @Nonnull PolarisConfigurationStore configurationStore) { + @Nonnull PolarisConfigurationStore configurationStore, + @Nonnull EntityTransformationEngine entityTransformationEngine) { this.diagnostics = diagnostics; this.configurationStore = configurationStore; + this.entityTransformationEngine = entityTransformationEngine; } protected abstract StoreType createBackingStore(@Nonnull PolarisDiagnostics diagnostics); @@ -139,7 +143,8 @@ public Map purgeRealms(Iterable realms) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); TransactionalPersistence session = getOrCreateSessionSupplier(realmContext).get(); - PolarisCallContext callContext = new PolarisCallContext(realmContext, session, diagServices); + PolarisCallContext callContext = + new PolarisCallContext(realmContext, session, diagServices, entityTransformationEngine); BaseResult result = metaStoreManager.purge(callContext); results.put(realm, result); @@ -214,7 +219,8 @@ private PrincipalSecretsResult bootstrapServiceAndCreatePolarisPrincipalForRealm new PolarisCallContext( realmContext, sessionSupplierMap.get(realmContext.getRealmIdentifier()).get(), - diagServices); + diagServices, + entityTransformationEngine); if (CallContext.getCurrentContext() == null) { CallContext.setCurrentContext(polarisContext); } @@ -263,7 +269,8 @@ private void checkPolarisServiceBootstrappedForRealm( new PolarisCallContext( realmContext, sessionSupplierMap.get(realmContext.getRealmIdentifier()).get(), - diagServices); + diagServices, + entityTransformationEngine); if (CallContext.getCurrentContext() == null) { CallContext.setCurrentContext(polarisContext); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index a81566c99a..6137013a78 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -47,6 +47,8 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; +import org.apache.polaris.core.entity.transformation.TransformationPoint; import org.apache.polaris.core.persistence.BaseMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; @@ -477,6 +479,14 @@ private void revokeGrantRecord( ms.persistStorageIntegrationIfNeededInCurrentTxn(callCtx, catalog, integration); + // CATALOG_PRE_PERSIST transformation point + EntityTransformationEngine entityTransformationEngine = callCtx.getEntityTransformationEngine(); + if (entityTransformationEngine != null) { + catalog = + entityTransformationEngine.applyTransformers( + TransformationPoint.CATALOG_PRE_PERSIST, catalog); + } + // now create and persist new catalog entity this.persistNewEntity(callCtx, ms, catalog); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapAtomicOperationMetaStoreManagerTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapAtomicOperationMetaStoreManagerTest.java index f89615cf10..e796d2c537 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapAtomicOperationMetaStoreManagerTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapAtomicOperationMetaStoreManagerTest.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.transactional.TreeMapMetaStore; import org.apache.polaris.core.persistence.transactional.TreeMapTransactionalPersistenceImpl; import org.mockito.Mockito; @@ -41,6 +42,7 @@ public PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS), diagServices, new PolarisConfigurationStore() {}, + new NoOpEntityTransformationEngine(), timeSource.withZone(ZoneId.systemDefault())); return new PolarisTestMetaStoreManager(new AtomicOperationMetaStoreManager(), callCtx); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java index 49a2bfcc00..67a994db81 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl; import org.apache.polaris.core.persistence.transactional.TreeMapMetaStore; import org.apache.polaris.core.persistence.transactional.TreeMapTransactionalPersistenceImpl; @@ -41,6 +42,7 @@ public PolarisTestMetaStoreManager createPolarisTestMetaStoreManager() { new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS), diagServices, new PolarisConfigurationStore() {}, + new NoOpEntityTransformationEngine(), timeSource.withZone(ZoneId.systemDefault())); return new PolarisTestMetaStoreManager(new TransactionalMetaStoreManagerImpl(), callCtx); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java index d72a199ae7..344e312d4b 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java @@ -22,6 +22,8 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.transactional.TransactionalMetaStoreManagerImpl; import org.apache.polaris.core.persistence.transactional.TreeMapMetaStore; import org.apache.polaris.core.persistence.transactional.TreeMapTransactionalPersistenceImpl; @@ -40,7 +42,10 @@ protected PolarisCallContext callCtx() { TreeMapMetaStore store = new TreeMapMetaStore(diagServices); TreeMapTransactionalPersistenceImpl metaStore = new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS); - callCtx = new PolarisCallContext(() -> "testRealm", metaStore, diagServices); + EntityTransformationEngine entityTransformationEngine = new NoOpEntityTransformationEngine(); + callCtx = + new PolarisCallContext( + () -> "testRealm", metaStore, diagServices, entityTransformationEngine); } return callCtx; } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java index 945f1ccb6b..ee148f7de8 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/cache/InMemoryEntityCacheTest.java @@ -33,6 +33,8 @@ import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; @@ -65,6 +67,9 @@ public class InMemoryEntityCacheTest { // the meta store manager private final PolarisMetaStoreManager metaStoreManager; + // the entity transformation engine + private final EntityTransformationEngine entityTransformationEngine; + /** * Initialize and create the test metadata * @@ -91,7 +96,10 @@ public InMemoryEntityCacheTest() { diagServices = new PolarisDefaultDiagServiceImpl(); store = new TreeMapMetaStore(diagServices); metaStore = new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS); - callCtx = new PolarisCallContext(() -> "testRealm", metaStore, diagServices); + entityTransformationEngine = new NoOpEntityTransformationEngine(); + callCtx = + new PolarisCallContext( + () -> "testRealm", metaStore, diagServices, entityTransformationEngine); metaStoreManager = new TransactionalMetaStoreManagerImpl(); // bootstrap the mata store with our test schema diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java index e008abf74b..749a39a645 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/BaseStorageIntegrationTest.java @@ -22,12 +22,16 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.BasePersistence; import org.mockito.Mockito; public abstract class BaseStorageIntegrationTest { protected CallContext newCallContext() { return new PolarisCallContext( - () -> "realm", Mockito.mock(BasePersistence.class), Mockito.mock(PolarisDiagnostics.class)); + () -> "realm", + Mockito.mock(BasePersistence.class), + Mockito.mock(PolarisDiagnostics.class), + new NoOpEntityTransformationEngine()); } } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java index e679d3e32a..daa14bdf44 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/InMemoryStorageIntegrationTest.java @@ -30,6 +30,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -106,6 +107,7 @@ public void testValidateAccessToLocationsWithWildcard() { return (T) config.get(configName); } }, + new NoOpEntityTransformationEngine(), Clock.systemUTC()); CallContext.setCurrentContext(polarisCallContext); Map> result = diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index 3d889b7657..4491ad57b4 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -37,6 +37,8 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; import org.apache.polaris.core.persistence.dao.entity.BaseResult; @@ -70,7 +72,10 @@ public StorageCredentialCacheTest() { // to interact with the metastore TransactionalPersistence metaStore = new TreeMapTransactionalPersistenceImpl(store, Mockito.mock(), RANDOM_SECRETS); - callCtx = new PolarisCallContext(() -> "testRealm", metaStore, diagServices); + EntityTransformationEngine entityTransformationEngine = new NoOpEntityTransformationEngine(); + callCtx = + new PolarisCallContext( + () -> "testRealm", metaStore, diagServices, entityTransformationEngine); metaStoreManager = Mockito.mock(PolarisMetaStoreManager.class); storageCredentialCache = newStorageCredentialCache(); } diff --git a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java index 07ad023629..755fbc9163 100644 --- a/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java +++ b/quarkus/admin/src/main/java/org/apache/polaris/admintool/config/QuarkusProducers.java @@ -28,6 +28,7 @@ import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -71,6 +72,12 @@ PolarisStorageIntegration getStorageIntegrationForConfig( }; } + @Produces + public EntityTransformationEngine entityTransformationEngine() { + // An entity transformation engine is not required when running the admin tool. + return ((transformationPoint, entity) -> entity); + } + @Produces public PolarisConfigurationStore configurationStore() { // A configuration store is not required when running the admin tool. diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index 592501f76d..350aea6995 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -188,6 +188,9 @@ polaris.oidc.principal-roles-mapper.type=default # polaris.storage.gcp.token=token # polaris.storage.gcp.lifespan=PT1H +# Polaris Entity Transformation Config +polaris.entity-transformation.transformers=no-op + quarkus.arc.ignored-split-packages=\ org.apache.polaris.service.catalog.api,\ org.apache.polaris.service.catalog.api.impl,\ diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index e1b197005a..ea1f458d0d 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -42,6 +42,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; @@ -127,11 +128,17 @@ public CallContext polarisCallContext( PolarisDiagnostics diagServices, PolarisConfigurationStore configurationStore, MetaStoreManagerFactory metaStoreManagerFactory, + EntityTransformationEngine entityTransformationEngine, Clock clock) { BasePersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); return new PolarisCallContext( - realmContext, metaStoreSession, diagServices, configurationStore, clock); + realmContext, + metaStoreSession, + diagServices, + configurationStore, + entityTransformationEngine, + clock); } // Polaris service beans - selected from @Identifier-annotated beans diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/entity/transformation/QuarkusEntityTransformationConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/entity/transformation/QuarkusEntityTransformationConfiguration.java new file mode 100644 index 0000000000..5d377000f4 --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/entity/transformation/QuarkusEntityTransformationConfiguration.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.quarkus.entity.transformation; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import java.util.List; +import java.util.Optional; +import org.apache.polaris.core.entity.transformation.EntityTransformer; +import org.apache.polaris.service.entity.transformation.EntityTransformationConfiguration; + +/** + * Quarkus-specific configuration interface for entity transformation behavior. + * + *

This configuration determines which {@link EntityTransformer}s are applied during entity + * transformation and in what order. Only the listed transformers will be used, and they will be + * executed sequentially as configured. + * + *

If no transformers are specified, entity transformation is effectively disabled. + */ +@StaticInitSafe +@ConfigMapping(prefix = "polaris.entity-transformation") +public interface QuarkusEntityTransformationConfiguration + extends EntityTransformationConfiguration { + @Override + Optional> transformers(); +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java index 47128c86d1..f7daa94e71 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java @@ -75,6 +75,7 @@ public void setup() { .get(), fakeServices.polarisDiagnostics(), fakeServices.configurationStore(), + fakeServices.entityTransformationEngine(), Mockito.mock(Clock.class)); CallContext.setCurrentContext(polarisCallContext); services = @@ -188,7 +189,8 @@ private PolarisCallContext setupCallContext(PolarisMetaStoreManager metaStoreMan return new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - services.polarisDiagnostics()); + services.polarisDiagnostics(), + services.entityTransformationEngine()); } private PolarisAdminService setupPolarisAdminService( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 1c232bf54c..0dbf57f837 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -68,6 +68,8 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -206,6 +208,7 @@ public Map getConfigOverrides() { protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; protected PolarisMetaStoreManager metaStoreManager; + protected EntityTransformationEngine entityTransformationEngine; protected UserSecretsManager userSecretsManager; protected TransactionalPersistence metaStoreSession; protected PolarisBaseEntity catalogEntity; @@ -232,6 +235,7 @@ public void before(TestInfo testInfo) { QuarkusMock.installMockForType(realmContext, RealmContext.class); metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); userSecretsManager = userSecretsManagerFactory.getOrCreateUserSecretsManager(realmContext); + entityTransformationEngine = new NoOpEntityTransformationEngine(); polarisAuthorizer = new PolarisAuthorizerImpl(configurationStore); @@ -241,6 +245,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityTransformationEngine, clock); this.entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java index 46d2950b91..b633050ef8 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTRSAKeyPairTest.java @@ -61,7 +61,7 @@ public void testSuccessfulTokenGeneration() throws Exception { final String scope = "PRINCIPAL_ROLE:TEST"; PolarisCallContext polarisCallContext = - new PolarisCallContext(null, null, null, configurationStore, null); + new PolarisCallContext(null, null, null, configurationStore, null, null); PolarisMetaStoreManager metastoreManager = Mockito.mock(PolarisMetaStoreManager.class); String mainSecret = "client-secret"; PolarisPrincipalSecrets principalSecrets = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java index 6b51f6eda3..9a6c40c8f5 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/auth/JWTSymmetricKeyGeneratorTest.java @@ -45,7 +45,8 @@ public class JWTSymmetricKeyGeneratorTest { /** Sanity test to verify that we can generate a token */ @Test public void testJWTSymmetricKeyGenerator() { - PolarisCallContext polarisCallContext = new PolarisCallContext(null, null, null, null, null); + PolarisCallContext polarisCallContext = + new PolarisCallContext(null, null, null, null, null, null); PolarisMetaStoreManager metastoreManager = Mockito.mock(PolarisMetaStoreManager.class); String mainSecret = "test_secret"; String clientId = "test_client_id"; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 614fab7255..51dd8eb8f6 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -89,6 +89,7 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.TaskEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -210,6 +211,7 @@ public Map getConfigOverrides() { @Inject UserSecretsManagerFactory userSecretsManagerFactory; @Inject PolarisDiagnostics diagServices; @Inject PolarisEventListener polarisEventListener; + @Inject EntityTransformationEngine entityTransformationEngine; private IcebergCatalog catalog; private String realmName; @@ -257,6 +259,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityTransformationEngine, Clock.systemDefaultZone()); entityManager = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 6ea276cc09..7214f82abe 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -55,6 +55,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -132,6 +133,7 @@ public Map getConfigOverrides() { @Inject PolarisConfigurationStore configurationStore; @Inject PolarisDiagnostics diagServices; @Inject PolarisEventListener polarisEventListener; + @Inject EntityTransformationEngine entityTransformationEngine; private IcebergCatalog catalog; @@ -174,6 +176,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityTransformationEngine, Clock.systemDefaultZone()); PolarisEntityManager entityManager = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java index c9196f1750..af1b4cc78b 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java @@ -58,6 +58,7 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.table.GenericTableEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -128,6 +129,7 @@ public Map getConfigOverrides() { @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject PolarisDiagnostics diagServices; + @Inject EntityTransformationEngine entityTransformationEngine; private PolarisGenericTableCatalog genericTableCatalog; private IcebergCatalog icebergCatalog; @@ -173,6 +175,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityTransformationEngine, Clock.systemDefaultZone()); entityManager = new PolarisEntityManager( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java index c0d4b8b463..b7fb80c0c8 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java @@ -66,6 +66,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -158,6 +159,7 @@ public Map getConfigOverrides() { @Inject PolarisConfigurationStore configurationStore; @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject PolarisDiagnostics diagServices; + @Inject EntityTransformationEngine entityTransformationEngine; private PolicyCatalog policyCatalog; private IcebergCatalog icebergCatalog; @@ -199,6 +201,7 @@ public void before(TestInfo testInfo) { managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityTransformationEngine, Clock.systemDefaultZone()); entityManager = new PolarisEntityManager( diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java index 3da4bd2950..d488ed41e2 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/config/DefaultConfigurationStoreTest.java @@ -29,6 +29,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.service.config.DefaultConfigurationStore; import org.apache.polaris.service.config.FeaturesConfiguration; import org.assertj.core.api.Assertions; @@ -72,6 +73,7 @@ public Map getConfigOverrides() { @Inject PolarisConfigurationStore configurationStore; @Inject FeaturesConfiguration featuresConfiguration; + @Inject EntityTransformationEngine entityTransformationEngine; @BeforeEach public void before(TestInfo testInfo) { diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java index ed44510a9d..1a4c466763 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/entity/CatalogEntityTest.java @@ -31,6 +31,8 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.assertj.core.api.Assertions; @@ -47,11 +49,13 @@ public class CatalogEntityTest { public void setup() { MetaStoreManagerFactory metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); RealmContext realmContext = () -> "realm"; + EntityTransformationEngine entityTransformationEngine = new NoOpEntityTransformationEngine(); PolarisCallContext polarisCallContext = new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(() -> "realm").get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityTransformationEngine); this.callContext = polarisCallContext; CallContext.setCurrentContext(polarisCallContext); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/BatchFileCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/BatchFileCleanupTaskHandlerTest.java index 5a4de3109a..6e64809adf 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/BatchFileCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/BatchFileCleanupTaskHandlerTest.java @@ -51,6 +51,7 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisTaskConstants; import org.apache.polaris.core.entity.TaskEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -63,6 +64,7 @@ @QuarkusTest public class BatchFileCleanupTaskHandlerTest { @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject EntityTransformationEngine entityTransformationEngine; private final RealmContext realmContext = () -> "realmName"; private TaskFileIOSupplier buildTaskFileIOSupplier(FileIO fileIO) { @@ -94,7 +96,8 @@ public void testMetadataFileCleanup() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityTransformationEngine); FileIO fileIO = new InMemoryFileIO() { @Override @@ -207,7 +210,8 @@ public void testMetadataFileCleanupIfFileNotExist() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityTransformationEngine); CallContext.setCurrentContext(polarisCallContext); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); @@ -252,7 +256,8 @@ public void testCleanupWithRetries() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityTransformationEngine); CallContext.setCurrentContext(polarisCallContext); Map retryCounter = new HashMap<>(); FileIO fileIO = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java index e6ee6724e3..8ae13947e6 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java @@ -49,6 +49,7 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisTaskConstants; import org.apache.polaris.core.entity.TaskEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -61,6 +62,7 @@ @QuarkusTest class ManifestFileCleanupTaskHandlerTest { @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject EntityTransformationEngine entityTransformationEngine; private final RealmContext realmContext = () -> "realmName"; @@ -93,7 +95,8 @@ public void testCleanupFileNotExists() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityTransformationEngine); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); @@ -123,7 +126,8 @@ public void testCleanupFileManifestExistsDataFilesDontExist() throws IOException new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityTransformationEngine); CallContext.setCurrentContext(polarisCallContext); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); @@ -152,7 +156,8 @@ public void testCleanupFiles() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityTransformationEngine); CallContext.setCurrentContext(polarisCallContext); FileIO fileIO = new InMemoryFileIO() { @@ -198,7 +203,8 @@ public void testCleanupFilesWithRetries() throws IOException { new PolarisCallContext( realmContext, metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), - new PolarisDefaultDiagServiceImpl()); + new PolarisDefaultDiagServiceImpl(), + entityTransformationEngine); CallContext.setCurrentContext(polarisCallContext); Map retryCounter = new HashMap<>(); FileIO fileIO = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java index 5c968f7e88..90cf23ef89 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java @@ -53,6 +53,7 @@ import org.apache.polaris.core.entity.PolarisTaskConstants; import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; @@ -75,6 +76,7 @@ class TableCleanupTaskHandlerTest { @Inject MetaStoreManagerFactory metaStoreManagerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisDiagnostics diagServices; + @Inject EntityTransformationEngine entityTransformationEngine; private CallContext callContext; @@ -107,6 +109,7 @@ void setup() { metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, configurationStore, + entityTransformationEngine, Clock.systemDefaultZone()); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java index e9268befb7..df68a5f289 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java @@ -118,6 +118,7 @@ private PolarisPrincipalSecrets fetchAdminSecrets() { metaStoreSession, helper.diagServices, helper.configurationStore, + helper.entityTransformationEngine, helper.clock); try { PolarisMetaStoreManager metaStoreManager = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java index 7149291543..4eb394b29e 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestHelper.java @@ -24,6 +24,7 @@ import java.time.Clock; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.context.RealmContextResolver; import org.junit.jupiter.api.TestInfo; @@ -36,6 +37,7 @@ public class PolarisIntegrationTestHelper { @Inject ObjectMapper objectMapper; @Inject PolarisDiagnostics diagServices; @Inject PolarisConfigurationStore configurationStore; + @Inject EntityTransformationEngine entityTransformationEngine; @Inject Clock clock; public PolarisIntegrationTestFixture createFixture(TestEnvironment testEnv, TestInfo testInfo) { diff --git a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java index 62b113daea..6e589e8a96 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java @@ -28,6 +28,7 @@ import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.slf4j.Logger; @@ -47,6 +48,7 @@ public class DefaultCallContextResolver implements CallContextResolver { @Inject MetaStoreManagerFactory metaStoreManagerFactory; @Inject PolarisConfigurationStore configurationStore; @Inject PolarisDiagnostics diagnostics; + @Inject EntityTransformationEngine entityTransformationEngine; @Inject Clock clock; @Override @@ -67,6 +69,11 @@ public CallContext resolveCallContext( BasePersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); return new PolarisCallContext( - realmContext, metaStoreSession, diagnostics, configurationStore, clock); + realmContext, + metaStoreSession, + diagnostics, + configurationStore, + entityTransformationEngine, + clock); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/entity/transformation/AppliesTo.java b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/AppliesTo.java new file mode 100644 index 0000000000..2e2714814a --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/AppliesTo.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.entity.transformation; + +import jakarta.enterprise.util.AnnotationLiteral; +import jakarta.inject.Qualifier; +import java.lang.annotation.ElementType; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; +import org.apache.polaris.core.entity.transformation.EntityTransformer; +import org.apache.polaris.core.entity.transformation.TransformationPoint; + +/** + * Qualifier to mark an {@link EntityTransformer} as applicable to a specific {@link + * TransformationPoint}. + * + *

This is used by the {@link EntityTransformationEngine} to apply only relevant transformers + * based on context. + * + *

Supports being repeated on the same class to handle multiple transformation points. + */ +@Qualifier +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Repeatable(AppliesTos.class) +public @interface AppliesTo { + + /** The transformation point this transformer applies to. */ + TransformationPoint value(); + + /** Helper for creating {@link AppliesTo} qualifiers programmatically. */ + final class Literal extends AnnotationLiteral implements AppliesTo { + private final TransformationPoint value; + + public static Literal of(TransformationPoint value) { + return new Literal(value); + } + + private Literal(TransformationPoint value) { + this.value = value; + } + + @Override + public TransformationPoint value() { + return value; + } + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/entity/transformation/AppliesTos.java b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/AppliesTos.java new file mode 100644 index 0000000000..21259979a6 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/AppliesTos.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.entity.transformation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.apache.polaris.core.entity.transformation.EntityTransformer; +import org.apache.polaris.core.entity.transformation.TransformationPoint; + +/** + * Container annotation for repeating {@link AppliesTo}. + * + *

Allows an {@link EntityTransformer} to declare support for multiple {@link + * TransformationPoint}s. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface AppliesTos { + AppliesTo[] value(); +} diff --git a/service/common/src/main/java/org/apache/polaris/service/entity/transformation/EntityTransformationConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/EntityTransformationConfiguration.java new file mode 100644 index 0000000000..e924446ce2 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/EntityTransformationConfiguration.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.entity.transformation; + +import java.util.List; +import java.util.Optional; + +public interface EntityTransformationConfiguration { + Optional> transformers(); +} diff --git a/service/common/src/main/java/org/apache/polaris/service/entity/transformation/EntityTransformationEngineImpl.java b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/EntityTransformationEngineImpl.java new file mode 100644 index 0000000000..a22dc117c1 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/EntityTransformationEngineImpl.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.entity.transformation; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Instance; +import jakarta.inject.Inject; +import java.util.List; +import java.util.Objects; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; +import org.apache.polaris.core.entity.transformation.EntityTransformer; +import org.apache.polaris.core.entity.transformation.TransformationPoint; + +@ApplicationScoped +public class EntityTransformationEngineImpl implements EntityTransformationEngine { + + private final EntityTransformationConfiguration config; + private final Instance transformerInstances; + + @Inject + public EntityTransformationEngineImpl( + EntityTransformationConfiguration config, + @Any Instance transformerInstances) { + this.config = config; + this.transformerInstances = transformerInstances; + } + + @Override + public PolarisBaseEntity applyTransformers( + TransformationPoint transformationPoint, PolarisBaseEntity entity) { + PolarisBaseEntity result = entity; + + // Collect transformers in configured order, filtering only those applicable to the + // transformation point + List orderedtransformers = + config.transformers().orElse(List.of()).stream() + .map( + id -> { + // Resolve the transformer instance by ID + Instance matched = + transformerInstances.select(Identifier.Literal.of(id)); + if (!matched.isResolvable()) { + throw new IllegalStateException("No Entitytransformer found for ID: " + id); + } + // Filter by TransformationPoint via @AppliesTo + Instance filtered = + matched.select(AppliesTo.Literal.of(transformationPoint)); + return filtered.isResolvable() ? filtered.get() : null; + }) + .filter(Objects::nonNull) + .toList(); + + for (EntityTransformer transformer : orderedtransformers) { + result = transformer.apply(result); + } + + return result; + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/entity/transformation/NoOpEntityTransformer.java b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/NoOpEntityTransformer.java new file mode 100644 index 0000000000..0c21d00472 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/entity/transformation/NoOpEntityTransformer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.entity.transformation; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformer; + +/** + * A no-op implementation of {@link EntityTransformer} that returns the entity unchanged. + * + *

This can be used as a placeholder or for testing purposes when no transformer is required. + */ +@ApplicationScoped +@Identifier("no-op") +public class NoOpEntityTransformer implements EntityTransformer { + @Override + public PolarisBaseEntity apply(PolarisBaseEntity entity) { + return entity; + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java index 83908f0178..787f4c01cf 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryAtomicOperationMetaStoreManagerFactory.java @@ -23,6 +23,7 @@ import jakarta.inject.Inject; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.AtomicOperationMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -37,15 +38,16 @@ public class InMemoryAtomicOperationMetaStoreManagerFactory extends InMemoryPolarisMetaStoreManagerFactory { public InMemoryAtomicOperationMetaStoreManagerFactory() { - super(null, null, null); + super(null, null, null, null); } @Inject public InMemoryAtomicOperationMetaStoreManagerFactory( PolarisStorageIntegrationProvider storageIntegration, PolarisDiagnostics diagnostics, - PolarisConfigurationStore configurationStore) { - super(storageIntegration, diagnostics, configurationStore); + PolarisConfigurationStore configurationStore, + EntityTransformationEngine entityTransformationEngine) { + super(storageIntegration, diagnostics, configurationStore, entityTransformationEngine); } @Override diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 367a57de63..05c78a8763 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -31,6 +31,7 @@ import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.PolarisConfigurationStore; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; @@ -49,15 +50,16 @@ public class InMemoryPolarisMetaStoreManagerFactory private final Set bootstrappedRealms = new HashSet<>(); public InMemoryPolarisMetaStoreManagerFactory() { - this(null, null, null); + this(null, null, null, null); } @Inject public InMemoryPolarisMetaStoreManagerFactory( PolarisStorageIntegrationProvider storageIntegration, PolarisDiagnostics diagnostics, - PolarisConfigurationStore configurationStore) { - super(diagnostics, configurationStore); + PolarisConfigurationStore configurationStore, + EntityTransformationEngine entityTransformationEngine) { + super(diagnostics, configurationStore, entityTransformationEngine); this.storageIntegration = storageIntegration; } diff --git a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java index 9928e00d1f..3423ee42b9 100644 --- a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java @@ -142,6 +142,7 @@ FileIO loadFileIOInternal( testServices.metaStoreManagerFactory().getOrCreateSessionSupplier(realmContext).get(), testServices.polarisDiagnostics(), testServices.configurationStore(), + testServices.entityTransformationEngine(), Clock.systemUTC()); } diff --git a/service/common/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java b/service/common/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java index c67f7f172a..5f505b110e 100644 --- a/service/common/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/task/TaskExecutorImplTest.java @@ -50,7 +50,11 @@ void testEventsAreEmitted() { BasePersistence bp = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); PolarisCallContext polarisCallCtx = - new PolarisCallContext(realmContext, bp, testServices.polarisDiagnostics()); + new PolarisCallContext( + realmContext, + bp, + testServices.polarisDiagnostics(), + testServices.entityTransformationEngine()); // This task doesn't have a type so it won't be handle-able by a real handler. We register a // test TaskHandler below that can handle any task. diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 8f63ecfb83..26c52e4a9c 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -39,6 +39,8 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.entity.transformation.EntityTransformationEngine; +import org.apache.polaris.core.entity.transformation.NoOpEntityTransformationEngine; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; @@ -75,6 +77,7 @@ public record TestServices( IcebergRestConfigurationApi restConfigurationApi, PolarisConfigurationStore configurationStore, PolarisDiagnostics polarisDiagnostics, + EntityTransformationEngine entityTransformationEngine, RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, RealmContext realmContext, @@ -152,9 +155,13 @@ public TestServices build() { () -> stsClient, Optional.empty(), () -> GoogleCredentials.create(new AccessToken(GCP_ACCESS_TOKEN, new Date()))); + EntityTransformationEngine entityTransformationEngine = new NoOpEntityTransformationEngine(); InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory( - storageIntegrationProvider, polarisDiagnostics, configurationStore); + storageIntegrationProvider, + polarisDiagnostics, + configurationStore, + entityTransformationEngine); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(metaStoreManagerFactory) {}; UserSecretsManagerFactory userSecretsManagerFactory = @@ -168,6 +175,7 @@ public TestServices build() { metaStoreSession, polarisDiagnostics, configurationStore, + entityTransformationEngine, Clock.systemUTC()); PolarisEntityManager entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); @@ -264,6 +272,7 @@ public String getAuthenticationScheme() { restConfigurationApi, configurationStore, polarisDiagnostics, + entityTransformationEngine, realmEntityManagerFactory, metaStoreManagerFactory, realmContext, From eace12274a2998b9a3ea3e2834ae5f72fa80560c Mon Sep 17 00:00:00 2001 From: Rulin Xing Date: Mon, 16 Jun 2025 02:49:44 -0700 Subject: [PATCH 2/2] Ignore split-packages --- quarkus/defaults/src/main/resources/application.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index 350aea6995..d37cae14ea 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -201,6 +201,7 @@ quarkus.arc.ignored-split-packages=\ org.apache.polaris.service.quarkus.auth.external.mapping,\ org.apache.polaris.service.quarkus.auth.external.tenant,\ org.apache.polaris.service.quarkus.auth.internal,\ + org.apache.polaris.service.quarkus.entity.transformation,\ org.apache.polaris.service.quarkus.events,\ org.apache.polaris.service.quarkus.task,\ org.apache.polaris.service.quarkus.secrets,\