From efbb6661e93cbea1f14c09f5b45e0841fab89c60 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Mon, 24 Feb 2025 02:41:51 +0000 Subject: [PATCH 01/15] Remove all tightly coupled EntityCache dependencies in the main persistence stack Remove the EntityCacheEntry wrapper since the timestamp fields were never used anyways; instead the underlying Caffeine cache transparently handles access times, and the types of entries we cache are simply the existing ResolvedPolarisEntity. Remove interactions of business logic with explicit "cache entries", instead operating on ResolvedPolarisEntity. Fix the equals()/hashCode() behavior of PolarisEntity to be compatible with PolarisBaseEntity as intended. Improve code comments to explain the (current) relationship between PolarisEntity and PolarisBaseEntity, and clarify the behaviors in Resolver.java. Fully remove the PolarisRemoteCache interface and its methods. Add different methods that aren't cache-specific instead. --- .../polaris/core/entity/PolarisEntity.java | 52 ++-- .../persistence/PolarisMetaStoreManager.java | 203 ++++++++++++++- .../PolarisMetaStoreManagerImpl.java | 40 +-- .../persistence/ResolvedPolarisEntity.java | 67 ++++- .../TransactionWorkspaceMetaStoreManager.java | 12 +- .../core/persistence/cache/EntityCache.java | 79 +++--- .../persistence/cache/EntityCacheEntry.java | 129 ---------- .../cache/EntityCacheLookupResult.java | 7 +- .../persistence/cache/PolarisRemoteCache.java | 232 ------------------ .../resolver/PolarisResolutionManifest.java | 50 ++-- .../core/persistence/resolver/Resolver.java | 222 +++++++++-------- .../core/persistence/EntityCacheTest.java | 17 +- .../core/persistence/ResolverTest.java | 42 ++-- .../PolarisTestMetaStoreManager.java | 22 +- .../catalog/IcebergCatalogAdapter.java | 4 +- 15 files changed, 526 insertions(+), 652 deletions(-) delete mode 100644 polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCacheEntry.java delete mode 100644 polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/PolarisRemoteCache.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java index 5f0a7ad651..3ded968af2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java @@ -25,12 +25,23 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +/** + * For legacy reasons, this class is only a thin facade over PolarisBaseEntity's members/methods. No + * direct members should be added to this class; rather, they should reside in the PolarisBaseEntity + * and this class should just contain the relevant builder methods, etc. The intention when using + * this class is to use "immutable" semantics as much as possible, for example constructing new + * copies with the Builder pattern when "mutating" fields rather than ever chaing fields in-place. + * Currently, code that intends to operate directly on a PolarisBaseEntity may not adhere to + * immutability semantics, and may modify the entity in-place. + * + *

TODO: Combine this fully into PolarisBaseEntity, refactor all callsites to use strict + * immutability semantics, and remove all mutator methods from PolarisBaseEntity. + */ public class PolarisEntity extends PolarisBaseEntity { public static class NameAndId { @@ -227,41 +238,18 @@ public String toString() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof PolarisEntity)) return false; - PolarisEntity that = (PolarisEntity) o; - return catalogId == that.catalogId - && id == that.id - && parentId == that.parentId - && createTimestamp == that.createTimestamp - && dropTimestamp == that.dropTimestamp - && purgeTimestamp == that.purgeTimestamp - && lastUpdateTimestamp == that.lastUpdateTimestamp - && entityVersion == that.entityVersion - && grantRecordsVersion == that.grantRecordsVersion - && typeCode == that.typeCode - && subTypeCode == that.subTypeCode - && Objects.equals(name, that.name) - && Objects.equals(properties, that.properties) - && Objects.equals(internalProperties, that.internalProperties); + // Note: Keeping this here explicitly instead silently inheriting super.equals as a more + // prominent warning that the data members of this class *must not* diverge from those of + // PolarisBaseEntity. + return super.equals(o); } @Override public int hashCode() { - return Objects.hash( - typeCode, - subTypeCode, - catalogId, - id, - parentId, - name, - createTimestamp, - dropTimestamp, - purgeTimestamp, - lastUpdateTimestamp, - properties, - internalProperties, - entityVersion, - grantRecordsVersion); + // Note: Keeping this here explicitly instead silently inheriting super.hashCode as a more + // prominent warning that the data members of this class *must not* diverge from those of + // PolarisBaseEntity. + return super.hashCode(); } public static class Builder extends BaseBuilder { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 69652cb872..30c72a26a1 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -29,13 +29,15 @@ import org.apache.polaris.core.auth.PolarisGrantManager; import org.apache.polaris.core.auth.PolarisSecretsManager; import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.cache.PolarisRemoteCache; import org.apache.polaris.core.storage.PolarisCredentialVendor; /** @@ -43,10 +45,7 @@ * authorization. It uses the underlying persistent metastore to store and retrieve Polaris metadata */ public interface PolarisMetaStoreManager - extends PolarisSecretsManager, - PolarisGrantManager, - PolarisRemoteCache, - PolarisCredentialVendor { + extends PolarisSecretsManager, PolarisGrantManager, PolarisCredentialVendor { /** * Bootstrap the Polaris service, creating the root catalog, root principal, and associated @@ -688,4 +687,198 @@ DropEntityResult dropEntityIfExists( */ @Nonnull EntitiesResult loadTasks(@Nonnull PolarisCallContext callCtx, String executorId, int limit); + + /** Result of a loadEntitiesChangeTracking call */ + class ChangeTrackingResult extends BaseResult { + + // null if not success. Else, will be null if the grant to revoke was not found + private final List changeTrackingVersions; + + /** + * Constructor for an error + * + * @param errorCode error code, cannot be SUCCESS + * @param extraInformation extra information + */ + public ChangeTrackingResult( + @Nonnull BaseResult.ReturnStatus errorCode, @Nullable String extraInformation) { + super(errorCode, extraInformation); + this.changeTrackingVersions = null; + } + + /** + * Constructor for success + * + * @param changeTrackingVersions change tracking versions + */ + public ChangeTrackingResult( + @Nonnull List changeTrackingVersions) { + super(BaseResult.ReturnStatus.SUCCESS); + this.changeTrackingVersions = changeTrackingVersions; + } + + @JsonCreator + private ChangeTrackingResult( + @JsonProperty("returnStatus") @Nonnull BaseResult.ReturnStatus returnStatus, + @JsonProperty("extraInformation") String extraInformation, + @JsonProperty("changeTrackingVersions") + List changeTrackingVersions) { + super(returnStatus, extraInformation); + this.changeTrackingVersions = changeTrackingVersions; + } + + public List getChangeTrackingVersions() { + return changeTrackingVersions; + } + } + + /** + * Load change tracking information for a set of entities in one single shot and return for each + * the version for the entity itself and the version associated to its grant records. + * + * @param callCtx call context + * @param entityIds list of catalog/entity pair ids for which we need to efficiently load the + * version information, both entity version and grant records version. + * @return a list of version tracking information. Order in that returned list is the same as the + * input list. Some elements might be NULL if the entity has been purged. Not expected to fail + */ + @Nonnull + ChangeTrackingResult loadEntitiesChangeTracking( + @Nonnull PolarisCallContext callCtx, @Nonnull List entityIds); + + /** + * Represents an entity with its grants. If we "refresh" a previously fetched entity, we will only + * refresh the information which has changed, based on the version of the entity. + */ + class ResolvedEntityResult extends BaseResult { + + // the entity itself if it was loaded + private final @Nullable PolarisBaseEntity entity; + + // version for the grant records, in case the entity was not loaded + private final int grantRecordsVersion; + + private final @Nullable List entityGrantRecords; + + /** + * Constructor for an error + * + * @param errorCode error code, cannot be SUCCESS + * @param extraInformation extra information + */ + public ResolvedEntityResult( + @Nonnull BaseResult.ReturnStatus errorCode, @Nullable String extraInformation) { + super(errorCode, extraInformation); + this.entity = null; + this.entityGrantRecords = null; + this.grantRecordsVersion = 0; + } + + /** + * Constructor with success + * + * @param entity the main entity + * @param grantRecordsVersion the version of the grant records + * @param entityGrantRecords the list of grant records + */ + public ResolvedEntityResult( + @Nullable PolarisBaseEntity entity, + int grantRecordsVersion, + @Nullable List entityGrantRecords) { + super(BaseResult.ReturnStatus.SUCCESS); + this.entity = entity; + this.entityGrantRecords = entityGrantRecords; + this.grantRecordsVersion = grantRecordsVersion; + } + + @JsonCreator + public ResolvedEntityResult( + @JsonProperty("returnStatus") @Nonnull BaseResult.ReturnStatus returnStatus, + @JsonProperty("extraInformation") String extraInformation, + @Nullable @JsonProperty("entity") PolarisBaseEntity entity, + @JsonProperty("grantRecordsVersion") int grantRecordsVersion, + @Nullable @JsonProperty("entityGrantRecords") List entityGrantRecords) { + super(returnStatus, extraInformation); + this.entity = entity; + this.entityGrantRecords = entityGrantRecords; + this.grantRecordsVersion = grantRecordsVersion; + } + + public @Nullable PolarisBaseEntity getEntity() { + return entity; + } + + public int getGrantRecordsVersion() { + return grantRecordsVersion; + } + + public @Nullable List getEntityGrantRecords() { + return entityGrantRecords; + } + } + + /** + * Load a resolved entity, i.e. an entity definition and associated grant records, from the + * backend store. The entity is identified by its id (entity catalog id and id). + * + *

For entities that can be grantees, the associated grant records will include both the grant + * records for this entity as a grantee and for this entity as a securable. + * + * @param callCtx call context + * @param entityCatalogId id of the catalog for that entity + * @param entityId id of the entity + * @return result with entity and grants. Status will be ENTITY_NOT_FOUND if the entity was not + * found + */ + @Nonnull + ResolvedEntityResult loadResolvedEntityById( + @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId); + + /** + * Load a resolved entity, i.e. an entity definition and associated grant records, from the + * backend store. The entity is identified by its name. Will return NULL if the entity does not + * exist, i.e. has been purged or dropped. + * + *

For entities that can be grantees, the associated grant records will include both the grant + * records for this entity as a grantee and for this entity as a securable. + * + * @param callCtx call context + * @param entityCatalogId id of the catalog for that entity + * @param parentId the id of the parent of that entity + * @param entityType the type of this entity + * @param entityName the name of this entity + * @return result with entity and grants. Status will be ENTITY_NOT_FOUND if the entity was not + * found + */ + @Nonnull + ResolvedEntityResult loadResolvedEntityByName( + @Nonnull PolarisCallContext callCtx, + long entityCatalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + @Nonnull String entityName); + + /** + * Refresh a resolved entity from the backend store. Will return NULL if the entity does not + * exist, i.e. has been purged or dropped. Else, will determine what has changed based on the + * version information sent by the caller and will return only what has changed. + * + *

For entities that can be grantees, the associated grant records will include both the grant + * records for this entity as a grantee and for this entity as a securable. + * + * @param callCtx call context + * @param entityType type of the entity whose entity and grants we are refreshing + * @param entityCatalogId id of the catalog for that entity + * @param entityId the id of the entity to load + * @return result with entity and grants. Status will be ENTITY_NOT_FOUND if the entity was not + * found + */ + @Nonnull + ResolvedEntityResult refreshResolvedEntity( + @Nonnull PolarisCallContext callCtx, + int entityVersion, + int entityGrantRecordsVersion, + @Nonnull PolarisEntityType entityType, + long entityCatalogId, + long entityId); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index b90c7e51cf..98fd7789ca 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -2211,8 +2211,8 @@ public Map getInternalPropertyMap( return deserializeProperties(callCtx, internalPropStr); } - /** {@link #loadCachedEntryById(PolarisCallContext, long, long)} */ - private @Nonnull CachedEntryResult loadCachedEntryById( + /** {@link #loadResolvedEntityById(PolarisCallContext, long, long)} */ + private @Nonnull ResolvedEntityResult loadResolvedEntityById( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisMetaStoreSession ms, long entityCatalogId, @@ -2223,7 +2223,7 @@ public Map getInternalPropertyMap( // if entity not found, return null if (entity == null) { - return new CachedEntryResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); + return new ResolvedEntityResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); } // load the grant records @@ -2237,23 +2237,23 @@ public Map getInternalPropertyMap( } // return the result - return new CachedEntryResult(entity, entity.getGrantRecordsVersion(), grantRecords); + return new ResolvedEntityResult(entity, entity.getGrantRecordsVersion(), grantRecords); } /** {@inheritDoc} */ @Override - public @Nonnull CachedEntryResult loadCachedEntryById( + public @Nonnull ResolvedEntityResult loadResolvedEntityById( @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId) { // get metastore we should be using PolarisMetaStoreSession ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( - callCtx, () -> this.loadCachedEntryById(callCtx, ms, entityCatalogId, entityId)); + callCtx, () -> this.loadResolvedEntityById(callCtx, ms, entityCatalogId, entityId)); } - /** {@link #loadCachedEntryById(PolarisCallContext, long, long)} */ - private @Nonnull CachedEntryResult loadCachedEntryByName( + /** {@link #loadResolvedEntityById(PolarisCallContext, long, long)} */ + private @Nonnull ResolvedEntityResult loadResolvedEntityByName( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisMetaStoreSession ms, long entityCatalogId, @@ -2268,7 +2268,7 @@ public Map getInternalPropertyMap( // null if entity not found if (entity == null) { - return new CachedEntryResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); + return new ResolvedEntityResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); } // load the grant records @@ -2284,12 +2284,12 @@ public Map getInternalPropertyMap( } // return the result - return new CachedEntryResult(entity, entity.getGrantRecordsVersion(), grantRecords); + return new ResolvedEntityResult(entity, entity.getGrantRecordsVersion(), grantRecords); } /** {@inheritDoc} */ @Override - public @Nonnull CachedEntryResult loadCachedEntryByName( + public @Nonnull ResolvedEntityResult loadResolvedEntityByName( @Nonnull PolarisCallContext callCtx, long entityCatalogId, long parentId, @@ -2299,11 +2299,11 @@ public Map getInternalPropertyMap( PolarisMetaStoreSession ms = callCtx.getMetaStore(); // need to run inside a read transaction - CachedEntryResult result = + ResolvedEntityResult result = ms.runInReadTransaction( callCtx, () -> - this.loadCachedEntryByName( + this.loadResolvedEntityByName( callCtx, ms, entityCatalogId, parentId, entityType, entityName)); if (PolarisEntityConstants.getRootContainerName().equals(entityName) && entityType == PolarisEntityType.ROOT @@ -2347,14 +2347,14 @@ public Map getInternalPropertyMap( ms.runInReadTransaction( callCtx, () -> - this.loadCachedEntryByName( + this.loadResolvedEntityByName( callCtx, ms, entityCatalogId, parentId, entityType, entityName)); } return result; } /** {@inheritDoc} */ - private @Nonnull CachedEntryResult refreshCachedEntity( + private @Nonnull ResolvedEntityResult refreshResolvedEntity( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisMetaStoreSession ms, int entityVersion, @@ -2370,7 +2370,7 @@ public Map getInternalPropertyMap( // if null, the entity has been purged if (entityVersions == null) { - return new CachedEntryResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); + return new ResolvedEntityResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); } // load the entity if something changed @@ -2380,7 +2380,7 @@ public Map getInternalPropertyMap( // if not found, return null if (entity == null) { - return new CachedEntryResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); + return new ResolvedEntityResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); } } else { // entity has not changed, no need to reload it @@ -2402,12 +2402,12 @@ public Map getInternalPropertyMap( } // return the result - return new CachedEntryResult(entity, entityVersions.getGrantRecordsVersion(), grantRecords); + return new ResolvedEntityResult(entity, entityVersions.getGrantRecordsVersion(), grantRecords); } /** {@inheritDoc} */ @Override - public @Nonnull CachedEntryResult refreshCachedEntity( + public @Nonnull ResolvedEntityResult refreshResolvedEntity( @Nonnull PolarisCallContext callCtx, int entityVersion, int entityGrantRecordsVersion, @@ -2421,7 +2421,7 @@ public Map getInternalPropertyMap( return ms.runInReadTransaction( callCtx, () -> - this.refreshCachedEntity( + this.refreshResolvedEntity( callCtx, ms, entityVersion, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/ResolvedPolarisEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/ResolvedPolarisEntity.java index a091362ac2..7449b97395 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/ResolvedPolarisEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/ResolvedPolarisEntity.java @@ -18,12 +18,14 @@ */ package org.apache.polaris.core.persistence; -import com.google.common.collect.ImmutableList; import jakarta.annotation.Nonnull; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; public class ResolvedPolarisEntity { private final PolarisEntity entity; @@ -37,6 +39,56 @@ public class ResolvedPolarisEntity { // these are the grants like TABLE_READ_PROPERTIES, NAMESPACE_LIST, etc. private final List grantRecordsAsSecurable; + /** + * Constructor used when an entry is initially created after loading the entity and its grants + * from the backend. + * + * @param diagnostics diagnostic services + * @param entity the entity which has just been loaded + * @param grantRecords associated grant records, including grants for this entity as a securable + * as well as grants for this entity as a grantee if applicable + * @param grantsVersion version of the grants when they were loaded + */ + public ResolvedPolarisEntity( + @Nonnull PolarisDiagnostics diagnostics, + @Nonnull PolarisBaseEntity entity, + @Nonnull List grantRecords, + int grantsVersion) { + // validate not null + diagnostics.checkNotNull(entity, "entity_null"); + diagnostics.checkNotNull(grantRecords, "grant_records_null"); + + // we copy all attributes of the entity to avoid any contamination + this.entity = PolarisEntity.of(entity); + + // if only the grant records have been reloaded because they were changed, the entity will + // have an old version for those. Patch the entity if this is the case, as if we had reloaded it + if (this.entity.getGrantRecordsVersion() != grantsVersion) { + // remember the grants versions. For now grants should be loaded after the entity, so expect + // grants version to be same or higher + diagnostics.check( + this.entity.getGrantRecordsVersion() <= grantsVersion, + "grants_version_going_backward", + "entity={} grantsVersion={}", + entity, + grantsVersion); + + // patch grant records version + this.entity.setGrantRecordsVersion(grantsVersion); + } + + // Split the combined list of grant records into grantee vs securable grants since the main + // usage pattern is to get the two lists separately. + this.grantRecordsAsGrantee = + grantRecords.stream() + .filter(record -> record.getGranteeId() == entity.getId()) + .collect(Collectors.toList()); + this.grantRecordsAsSecurable = + grantRecords.stream() + .filter(record -> record.getSecurableId() == entity.getId()) + .collect(Collectors.toList()); + } + public ResolvedPolarisEntity( PolarisEntity entity, List grantRecordsAsGrantee, @@ -48,16 +100,15 @@ public ResolvedPolarisEntity( this.grantRecordsAsSecurable = grantRecordsAsSecurable; } - public ResolvedPolarisEntity(@Nonnull EntityCacheEntry cacheEntry) { - this.entity = PolarisEntity.of(cacheEntry.getEntity()); - this.grantRecordsAsGrantee = ImmutableList.copyOf(cacheEntry.getGrantRecordsAsGrantee()); - this.grantRecordsAsSecurable = ImmutableList.copyOf(cacheEntry.getGrantRecordsAsSecurable()); - } - public PolarisEntity getEntity() { return entity; } + public @Nonnull List getAllGrantRecords() { + return Stream.concat(grantRecordsAsGrantee.stream(), grantRecordsAsSecurable.stream()) + .collect(Collectors.toList()); + } + /** The grant records associated with this entity being the grantee of the record. */ public List getGrantRecordsAsGrantee() { return grantRecordsAsGrantee; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java index 44134751bc..0d7c72d298 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java @@ -335,16 +335,16 @@ public ValidateAccessResult validateAccessToLocations( } @Override - public CachedEntryResult loadCachedEntryById( + public ResolvedEntityResult loadResolvedEntityById( @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId) { callCtx .getDiagServices() - .fail("illegal_method_in_transaction_workspace", "loadCachedEntryById"); + .fail("illegal_method_in_transaction_workspace", "loadResolvedEntityById"); return null; } @Override - public CachedEntryResult loadCachedEntryByName( + public ResolvedEntityResult loadResolvedEntityByName( @Nonnull PolarisCallContext callCtx, long entityCatalogId, long parentId, @@ -352,12 +352,12 @@ public CachedEntryResult loadCachedEntryByName( @Nonnull String entityName) { callCtx .getDiagServices() - .fail("illegal_method_in_transaction_workspace", "loadCachedEntryByName"); + .fail("illegal_method_in_transaction_workspace", "loadResolvedEntityByName"); return null; } @Override - public CachedEntryResult refreshCachedEntity( + public ResolvedEntityResult refreshResolvedEntity( @Nonnull PolarisCallContext callCtx, int entityVersion, int entityGrantRecordsVersion, @@ -366,7 +366,7 @@ public CachedEntryResult refreshCachedEntity( long entityId) { callCtx .getDiagServices() - .fail("illegal_method_in_transaction_workspace", "refreshCachedEntity"); + .fail("illegal_method_in_transaction_workspace", "refreshResolvedEntity"); return null; } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java index 05efba7fb7..791c43b2c0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java @@ -31,7 +31,9 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.apache.polaris.core.persistence.cache.PolarisRemoteCache.CachedEntryResult; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager.ResolvedEntityResult; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; /** The entity cache, can be private or shared */ public class EntityCache { @@ -40,26 +42,26 @@ public class EntityCache { private EntityCacheMode cacheMode; // the meta store manager - private final PolarisRemoteCache polarisRemoteCache; + private final PolarisMetaStoreManager polarisMetaStoreManager; // Caffeine cache to keep entries by id - private final Cache byId; + private final Cache byId; // index by name - private final AbstractMap byName; + private final AbstractMap byName; /** * Constructor. Cache can be private or shared * - * @param polarisRemoteCache the meta store manager implementation + * @param polarisMetaStoreManager the meta store manager implementation */ - public EntityCache(@Nonnull PolarisRemoteCache polarisRemoteCache) { + public EntityCache(@Nonnull PolarisMetaStoreManager polarisMetaStoreManager) { // by name cache this.byName = new ConcurrentHashMap<>(); // When an entry is removed, we simply remove it from the byName map - RemovalListener removalListener = + RemovalListener removalListener = (key, value, cause) -> { if (value != null) { // compute name key @@ -80,7 +82,7 @@ public EntityCache(@Nonnull PolarisRemoteCache polarisRemoteCache) { .build(); // remember the meta store manager - this.polarisRemoteCache = polarisRemoteCache; + this.polarisMetaStoreManager = polarisMetaStoreManager; // enabled by default this.cacheMode = EntityCacheMode.ENABLE; @@ -91,7 +93,7 @@ public EntityCache(@Nonnull PolarisRemoteCache polarisRemoteCache) { * * @param cacheEntry cache entry to remove */ - public void removeCacheEntry(@Nonnull EntityCacheEntry cacheEntry) { + public void removeCacheEntry(@Nonnull ResolvedPolarisEntity cacheEntry) { // compute name key EntityCacheByNameKey nameKey = new EntityCacheByNameKey(cacheEntry.getEntity()); @@ -107,13 +109,13 @@ public void removeCacheEntry(@Nonnull EntityCacheEntry cacheEntry) { * * @param cacheEntry new cache entry */ - private void cacheNewEntry(@Nonnull EntityCacheEntry cacheEntry) { + private void cacheNewEntry(@Nonnull ResolvedPolarisEntity cacheEntry) { // compute name key EntityCacheByNameKey nameKey = new EntityCacheByNameKey(cacheEntry.getEntity()); // get old value if one exist - EntityCacheEntry oldCacheEntry = this.byId.getIfPresent(cacheEntry.getEntity().getId()); + ResolvedPolarisEntity oldCacheEntry = this.byId.getIfPresent(cacheEntry.getEntity().getId()); // put new entry, only if really newer one this.byId @@ -147,7 +149,7 @@ private void cacheNewEntry(@Nonnull EntityCacheEntry cacheEntry) { * @param oldValue old cache entry * @return true if the newer cache entry */ - private boolean isNewer(EntityCacheEntry newValue, EntityCacheEntry oldValue) { + private boolean isNewer(ResolvedPolarisEntity newValue, ResolvedPolarisEntity oldValue) { return (newValue.getEntity().getEntityVersion() > oldValue.getEntity().getEntityVersion() || newValue.getEntity().getGrantRecordsVersion() > oldValue.getEntity().getGrantRecordsVersion()); @@ -160,7 +162,7 @@ private boolean isNewer(EntityCacheEntry newValue, EntityCacheEntry oldValue) { * @param newCacheEntry new entry */ private void replaceCacheEntry( - @Nullable EntityCacheEntry oldCacheEntry, @Nonnull EntityCacheEntry newCacheEntry) { + @Nullable ResolvedPolarisEntity oldCacheEntry, @Nonnull ResolvedPolarisEntity newCacheEntry) { // need to remove old? if (oldCacheEntry != null) { @@ -175,8 +177,6 @@ private void replaceCacheEntry( // delete the old one assuming it has not been replaced by the above new entry this.removeCacheEntry(oldCacheEntry); - } else { - oldCacheEntry.updateLastAccess(); } } else { // write new one @@ -223,7 +223,7 @@ public void setCacheMode(EntityCacheMode cacheMode) { * @param entityId entity id * @return the cache entry or null if not found */ - public @Nullable EntityCacheEntry getEntityById(long entityId) { + public @Nullable ResolvedPolarisEntity getEntityById(long entityId) { return byId.getIfPresent(entityId); } @@ -233,7 +233,8 @@ public void setCacheMode(EntityCacheMode cacheMode) { * @param entityNameKey entity name key * @return the cache entry or null if not found */ - public @Nullable EntityCacheEntry getEntityByName(@Nonnull EntityCacheByNameKey entityNameKey) { + public @Nullable ResolvedPolarisEntity getEntityByName( + @Nonnull EntityCacheByNameKey entityNameKey) { return byName.get(entityNameKey); } @@ -249,7 +250,7 @@ public void setCacheMode(EntityCacheMode cacheMode) { * records should be reloaded if needed * @return the cache entry for the entity or null if the specified entity does not exist */ - public @Nullable EntityCacheEntry getAndRefreshIfNeeded( + public @Nullable ResolvedPolarisEntity getAndRefreshIfNeeded( @Nonnull PolarisCallContext callContext, @Nonnull PolarisBaseEntity entityToValidate, int entityMinVersion, @@ -259,13 +260,13 @@ public void setCacheMode(EntityCacheMode cacheMode) { PolarisEntityType entityType = entityToValidate.getType(); // first lookup the cache to find the existing cache entry - EntityCacheEntry existingCacheEntry = this.getEntityById(entityId); + ResolvedPolarisEntity existingCacheEntry = this.getEntityById(entityId); // the caller's fetched entity may have come from a stale lookup byName; we should consider // the existingCacheEntry to be the older of the two for purposes of invalidation to make // sure when we replaceCacheEntry we're also removing the old name if it's no longer valid EntityCacheByNameKey nameKey = new EntityCacheByNameKey(entityToValidate); - EntityCacheEntry existingCacheEntryByName = this.getEntityByName(nameKey); + ResolvedPolarisEntity existingCacheEntryByName = this.getEntityByName(nameKey); if (existingCacheEntryByName != null && existingCacheEntry != null && isNewer(existingCacheEntry, existingCacheEntryByName)) { @@ -273,7 +274,7 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { } // the new one to be returned - final EntityCacheEntry newCacheEntry; + final ResolvedPolarisEntity newCacheEntry; // see if we need to load or refresh that entity if (existingCacheEntry == null @@ -281,7 +282,7 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { || existingCacheEntry.getEntity().getGrantRecordsVersion() < entityGrantRecordsMinVersion) { // the refreshed entity - final CachedEntryResult refreshedCacheEntry; + final ResolvedEntityResult refreshedCacheEntry; // was not found in the cache? final PolarisBaseEntity entity; @@ -290,7 +291,8 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { if (existingCacheEntry == null) { // try to load it refreshedCacheEntry = - this.polarisRemoteCache.loadCachedEntryById(callContext, entityCatalogId, entityId); + this.polarisMetaStoreManager.loadResolvedEntityById( + callContext, entityCatalogId, entityId); if (refreshedCacheEntry.isSuccess()) { entity = refreshedCacheEntry.getEntity(); grantRecords = refreshedCacheEntry.getEntityGrantRecords(); @@ -301,7 +303,7 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { } else { // refresh it refreshedCacheEntry = - this.polarisRemoteCache.refreshCachedEntity( + this.polarisMetaStoreManager.refreshResolvedEntity( callContext, existingCacheEntry.getEntity().getEntityVersion(), existingCacheEntry.getEntity().getGrantRecordsVersion(), @@ -336,20 +338,13 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { // create new cache entry newCacheEntry = - new EntityCacheEntry( - callContext.getDiagServices(), - existingCacheEntry == null - ? System.nanoTime() - : existingCacheEntry.getCreatedOnNanoTimestamp(), - entity, - grantRecords, - grantRecordsVersion); + new ResolvedPolarisEntity( + callContext.getDiagServices(), entity, grantRecords, grantRecordsVersion); // insert cache entry this.replaceCacheEntry(existingCacheEntry, newCacheEntry); } else { // found it in the cache and it is up-to-date, simply return it - existingCacheEntry.updateLastAccess(); newCacheEntry = existingCacheEntry; } @@ -369,7 +364,7 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { @Nonnull PolarisCallContext callContext, long entityCatalogId, long entityId) { // if it exists, we are set - EntityCacheEntry entry = this.getEntityById(entityId); + ResolvedPolarisEntity entry = this.getEntityById(entityId); final boolean cacheHit; // we need to load it if it does not exist @@ -378,8 +373,8 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { cacheHit = false; // load it - CachedEntryResult result = - polarisRemoteCache.loadCachedEntryById(callContext, entityCatalogId, entityId); + ResolvedEntityResult result = + polarisMetaStoreManager.loadResolvedEntityById(callContext, entityCatalogId, entityId); // not found, exit if (!result.isSuccess()) { @@ -392,9 +387,8 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { .getDiagServices() .checkNotNull(result.getEntityGrantRecords(), "entity_grant_records_should_loaded"); entry = - new EntityCacheEntry( + new ResolvedPolarisEntity( callContext.getDiagServices(), - System.nanoTime(), result.getEntity(), result.getEntityGrantRecords(), result.getGrantRecordsVersion()); @@ -421,7 +415,7 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { @Nonnull PolarisCallContext callContext, @Nonnull EntityCacheByNameKey entityNameKey) { // if it exists, we are set - EntityCacheEntry entry = this.getEntityByName(entityNameKey); + ResolvedPolarisEntity entry = this.getEntityByName(entityNameKey); final boolean cacheHit; // we need to load it if it does not exist @@ -430,8 +424,8 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { cacheHit = false; // load it - CachedEntryResult result = - polarisRemoteCache.loadCachedEntryByName( + ResolvedEntityResult result = + polarisMetaStoreManager.loadResolvedEntityByName( callContext, entityNameKey.getCatalogId(), entityNameKey.getParentId(), @@ -451,9 +445,8 @@ && isNewer(existingCacheEntry, existingCacheEntryByName)) { // if found, setup entry entry = - new EntityCacheEntry( + new ResolvedPolarisEntity( callContext.getDiagServices(), - System.nanoTime(), result.getEntity(), result.getEntityGrantRecords(), result.getGrantRecordsVersion()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCacheEntry.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCacheEntry.java deleted file mode 100644 index ca1a7e4e33..0000000000 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCacheEntry.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.core.persistence.cache; - -import com.google.common.collect.ImmutableList; -import jakarta.annotation.Nonnull; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisGrantRecord; - -/** An entry in our entity cache. Note, this is fully immutable */ -public class EntityCacheEntry { - - // epoch time (ns) when the cache entry was added to the cache - private final long createdOnNanoTimestamp; - - // epoch time (ns) when the cache entry was added to the cache - private long lastAccessedNanoTimestamp; - - // the entity which have been cached. - private final PolarisBaseEntity entity; - - // grants associated to this entity, for a principal, a principal role, or a catalog role these - // are role usage - // grants on that entity. For a catalog securable (i.e. a catalog, namespace, or table_like - // securable), these are - // the grants on this securable. - private final List grantRecords; - - /** - * Constructor used when an entry is initially created after loading the entity and its grants - * from the backend. - * - * @param diagnostics diagnostic services - * @param createdOnNanoTimestamp when the entity was created - * @param entity the entity which has just been loaded - * @param grantRecords associated grant records, including grants for this entity as a securable - * as well as grants for this entity as a grantee if applicable - * @param grantsVersion version of the grants when they were loaded - */ - EntityCacheEntry( - @Nonnull PolarisDiagnostics diagnostics, - long createdOnNanoTimestamp, - @Nonnull PolarisBaseEntity entity, - @Nonnull List grantRecords, - int grantsVersion) { - // validate not null - diagnostics.checkNotNull(entity, "entity_null"); - diagnostics.checkNotNull(grantRecords, "grant_records_null"); - - // when this entry has been created - this.createdOnNanoTimestamp = createdOnNanoTimestamp; - - // last accessed time is now - this.lastAccessedNanoTimestamp = System.nanoTime(); - - // we copy all attributes of the entity to avoid any contamination - this.entity = new PolarisBaseEntity(entity); - - // if only the grant records have been reloaded because they were changed, the entity will - // have an old version for those. Patch the entity if this is the case, as if we had reloaded it - if (this.entity.getGrantRecordsVersion() != grantsVersion) { - // remember the grants versions. For now grants should be loaded after the entity, so expect - // grants version to be same or higher - diagnostics.check( - this.entity.getGrantRecordsVersion() <= grantsVersion, - "grants_version_going_backward", - "entity={} grantsVersion={}", - entity, - grantsVersion); - - // patch grant records version - this.entity.setGrantRecordsVersion(grantsVersion); - } - - // the grants - this.grantRecords = ImmutableList.copyOf(grantRecords); - } - - public long getCreatedOnNanoTimestamp() { - return createdOnNanoTimestamp; - } - - public long getLastAccessedNanoTimestamp() { - return lastAccessedNanoTimestamp; - } - - public @Nonnull PolarisBaseEntity getEntity() { - return entity; - } - - public @Nonnull List getAllGrantRecords() { - return grantRecords; - } - - public @Nonnull List getGrantRecordsAsGrantee() { - return grantRecords.stream() - .filter(record -> record.getGranteeId() == entity.getId()) - .collect(Collectors.toList()); - } - - public @Nonnull List getGrantRecordsAsSecurable() { - return grantRecords.stream() - .filter(record -> record.getSecurableId() == entity.getId()) - .collect(Collectors.toList()); - } - - public void updateLastAccess() { - this.lastAccessedNanoTimestamp = System.nanoTime(); - } -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCacheLookupResult.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCacheLookupResult.java index a4677baa92..415fdd29b0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCacheLookupResult.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCacheLookupResult.java @@ -19,23 +19,24 @@ package org.apache.polaris.core.persistence.cache; import jakarta.annotation.Nullable; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; /** Result of a lookup operation */ public class EntityCacheLookupResult { // if not null, we found the entity and this is the entry. If not found, the entity was dropped or // does not exist - private final @Nullable EntityCacheEntry cacheEntry; + private final @Nullable ResolvedPolarisEntity cacheEntry; // true if the entity was found in the cache private final boolean cacheHit; - public EntityCacheLookupResult(@Nullable EntityCacheEntry cacheEntry, boolean cacheHit) { + public EntityCacheLookupResult(@Nullable ResolvedPolarisEntity cacheEntry, boolean cacheHit) { this.cacheEntry = cacheEntry; this.cacheHit = cacheHit; } - public @Nullable EntityCacheEntry getCacheEntry() { + public @Nullable ResolvedPolarisEntity getCacheEntry() { return cacheEntry; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/PolarisRemoteCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/PolarisRemoteCache.java deleted file mode 100644 index e45cc8b1df..0000000000 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/PolarisRemoteCache.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.core.persistence.cache; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.annotation.Nonnull; -import jakarta.annotation.Nullable; -import java.util.List; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; -import org.apache.polaris.core.entity.PolarisEntityId; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.apache.polaris.core.persistence.BaseResult; - -/** - * Interface to the remote entity cache. This allows the local cache to detect remote entity changes - * and refresh the local copies where necessary. - */ -public interface PolarisRemoteCache { - /** - * Load change tracking information for a set of entities in one single shot and return for each - * the version for the entity itself and the version associated to its grant records. - * - * @param callCtx call context - * @param entityIds list of catalog/entity pair ids for which we need to efficiently load the - * version information, both entity version and grant records version. - * @return a list of version tracking information. Order in that returned list is the same as the - * input list. Some elements might be NULL if the entity has been purged. Not expected to fail - */ - @Nonnull - ChangeTrackingResult loadEntitiesChangeTracking( - @Nonnull PolarisCallContext callCtx, @Nonnull List entityIds); - - /** - * Load a cached entry, i.e. an entity definition and associated grant records, from the backend - * store. The entity is identified by its id (entity catalog id and id). - * - *

For entities that can be grantees, the associated grant records will include both the grant - * records for this entity as a grantee and for this entity as a securable. - * - * @param callCtx call context - * @param entityCatalogId id of the catalog for that entity - * @param entityId id of the entity - * @return cached entry for this entity. Status will be ENTITY_NOT_FOUND if the entity was not - * found - */ - @Nonnull - CachedEntryResult loadCachedEntryById( - @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId); - - /** - * Load a cached entry, i.e. an entity definition and associated grant records, from the backend - * store. The entity is identified by its name. Will return NULL if the entity does not exist, - * i.e. has been purged or dropped. - * - *

For entities that can be grantees, the associated grant records will include both the grant - * records for this entity as a grantee and for this entity as a securable. - * - * @param callCtx call context - * @param entityCatalogId id of the catalog for that entity - * @param parentId the id of the parent of that entity - * @param entityType the type of this entity - * @param entityName the name of this entity - * @return cached entry for this entity. Status will be ENTITY_NOT_FOUND if the entity was not - * found - */ - @Nonnull - CachedEntryResult loadCachedEntryByName( - @Nonnull PolarisCallContext callCtx, - long entityCatalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull String entityName); - - /** - * Refresh a cached entity from the backend store. Will return NULL if the entity does not exist, - * i.e. has been purged or dropped. Else, will determine what has changed based on the version - * information sent by the caller and will return only what has changed. - * - *

For entities that can be grantees, the associated grant records will include both the grant - * records for this entity as a grantee and for this entity as a securable. - * - * @param callCtx call context - * @param entityType type of the entity whose cached entry we are refreshing - * @param entityCatalogId id of the catalog for that entity - * @param entityId the id of the entity to load - * @return cached entry for this entity. Status will be ENTITY_NOT_FOUND if the entity was not * - * found - */ - @Nonnull - CachedEntryResult refreshCachedEntity( - @Nonnull PolarisCallContext callCtx, - int entityVersion, - int entityGrantRecordsVersion, - @Nonnull PolarisEntityType entityType, - long entityCatalogId, - long entityId); - - /** Result of a loadEntitiesChangeTracking call */ - class ChangeTrackingResult extends BaseResult { - - // null if not success. Else, will be null if the grant to revoke was not found - private final List changeTrackingVersions; - - /** - * Constructor for an error - * - * @param errorCode error code, cannot be SUCCESS - * @param extraInformation extra information - */ - public ChangeTrackingResult( - @Nonnull BaseResult.ReturnStatus errorCode, @Nullable String extraInformation) { - super(errorCode, extraInformation); - this.changeTrackingVersions = null; - } - - /** - * Constructor for success - * - * @param changeTrackingVersions change tracking versions - */ - public ChangeTrackingResult( - @Nonnull List changeTrackingVersions) { - super(BaseResult.ReturnStatus.SUCCESS); - this.changeTrackingVersions = changeTrackingVersions; - } - - @JsonCreator - private ChangeTrackingResult( - @JsonProperty("returnStatus") @Nonnull BaseResult.ReturnStatus returnStatus, - @JsonProperty("extraInformation") String extraInformation, - @JsonProperty("changeTrackingVersions") - List changeTrackingVersions) { - super(returnStatus, extraInformation); - this.changeTrackingVersions = changeTrackingVersions; - } - - public List getChangeTrackingVersions() { - return changeTrackingVersions; - } - } - - /** - * Represents an entry in the cache. If we refresh a cached entry, we will only refresh the - * information which have changed, based on the version of the entity - */ - class CachedEntryResult extends BaseResult { - - // the entity itself if it was loaded - private final @Nullable PolarisBaseEntity entity; - - // version for the grant records, in case the entity was not loaded - private final int grantRecordsVersion; - - private final @Nullable List entityGrantRecords; - - /** - * Constructor for an error - * - * @param errorCode error code, cannot be SUCCESS - * @param extraInformation extra information - */ - public CachedEntryResult( - @Nonnull BaseResult.ReturnStatus errorCode, @Nullable String extraInformation) { - super(errorCode, extraInformation); - this.entity = null; - this.entityGrantRecords = null; - this.grantRecordsVersion = 0; - } - - /** - * Constructor with success - * - * @param entity the entity for that cached entry - * @param grantRecordsVersion the version of the grant records - * @param entityGrantRecords the list of grant records - */ - public CachedEntryResult( - @Nullable PolarisBaseEntity entity, - int grantRecordsVersion, - @Nullable List entityGrantRecords) { - super(BaseResult.ReturnStatus.SUCCESS); - this.entity = entity; - this.entityGrantRecords = entityGrantRecords; - this.grantRecordsVersion = grantRecordsVersion; - } - - @JsonCreator - public CachedEntryResult( - @JsonProperty("returnStatus") @Nonnull BaseResult.ReturnStatus returnStatus, - @JsonProperty("extraInformation") String extraInformation, - @Nullable @JsonProperty("entity") PolarisBaseEntity entity, - @JsonProperty("grantRecordsVersion") int grantRecordsVersion, - @Nullable @JsonProperty("entityGrantRecords") List entityGrantRecords) { - super(returnStatus, extraInformation); - this.entity = entity; - this.entityGrantRecords = entityGrantRecords; - this.grantRecordsVersion = grantRecordsVersion; - } - - public @Nullable PolarisBaseEntity getEntity() { - return entity; - } - - public int getGrantRecordsVersion() { - return grantRecordsVersion; - } - - public @Nullable List getEntityGrantRecords() { - return entityGrantRecords; - } - } -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java index 2ab0140a07..d37e2ce2ff 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java @@ -39,7 +39,6 @@ import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -211,23 +210,22 @@ public PolarisResolvedPathWrapper getPassthroughResolvedPath(Object key) { return null; } - List resolvedPath = passthroughResolver.getResolvedPath(); + List resolvedPath = passthroughResolver.getResolvedPath(); if (requestedPath.isOptional()) { if (resolvedPath.size() != requestedPath.getEntityNames().size()) { LOGGER.debug( "Returning null for key {} due to size mismatch from getPassthroughResolvedPath " + "resolvedPath: {}, requestedPath.getEntityNames(): {}", key, - resolvedPath.stream().map(ResolvedPolarisEntity::new).collect(Collectors.toList()), + resolvedPath, requestedPath.getEntityNames()); return null; } } List resolvedEntities = new ArrayList<>(); - resolvedEntities.add( - new ResolvedPolarisEntity(passthroughResolver.getResolvedReferenceCatalog())); - resolvedPath.forEach(cacheEntry -> resolvedEntities.add(new ResolvedPolarisEntity(cacheEntry))); + resolvedEntities.add(passthroughResolver.getResolvedReferenceCatalog()); + resolvedPath.forEach(resolvedEntity -> resolvedEntities.add(resolvedEntity)); LOGGER.debug( "Returning resolvedEntities from getPassthroughResolvedPath: {}", resolvedEntities); return new PolarisResolvedPathWrapper(resolvedEntities); @@ -255,11 +253,11 @@ public PolarisResolvedPathWrapper getPassthroughResolvedPath( public Set getAllActivatedCatalogRoleAndPrincipalRoles() { Set activatedRoles = new HashSet<>(); primaryResolver.getResolvedCallerPrincipalRoles().stream() - .map(EntityCacheEntry::getEntity) + .map(ResolvedPolarisEntity::getEntity) .forEach(activatedRoles::add); if (primaryResolver.getResolvedCatalogRoles() != null) { primaryResolver.getResolvedCatalogRoles().values().stream() - .map(EntityCacheEntry::getEntity) + .map(ResolvedPolarisEntity::getEntity) .forEach(activatedRoles::add); } return activatedRoles; @@ -268,7 +266,7 @@ public Set getAllActivatedCatalogRoleAndPrincipalRoles() { public Set getAllActivatedPrincipalRoleEntities() { Set activatedEntities = new HashSet<>(); primaryResolver.getResolvedCallerPrincipalRoles().stream() - .map(EntityCacheEntry::getEntity) + .map(ResolvedPolarisEntity::getEntity) .forEach(activatedEntities::add); return activatedEntities; } @@ -282,14 +280,14 @@ private ResolvedPolarisEntity getResolvedRootContainerEntity() { if (primaryResolverStatus.getStatus() != ResolverStatus.StatusEnum.SUCCESS) { return null; } - EntityCacheEntry resolvedCacheEntry = + ResolvedPolarisEntity resolvedEntity = primaryResolver.getResolvedEntity( PolarisEntityType.ROOT, PolarisEntityConstants.getRootContainerName()); - if (resolvedCacheEntry == null) { + if (resolvedEntity == null) { LOGGER.debug("Failed to find rootContainer, so using simulated rootContainer instead."); return simulatedResolvedRootContainerEntity; } - return new ResolvedPolarisEntity(resolvedCacheEntry); + return resolvedEntity; } public PolarisResolvedPathWrapper getResolvedRootContainerEntityAsPath() { @@ -302,8 +300,8 @@ public PolarisResolvedPathWrapper getResolvedReferenceCatalogEntity( // a callsite failed to incorporate a reference catalog into its authorization flow but is // still trying to perform operations on the (nonexistence) reference catalog. diagnostics.checkNotNull(catalogName, "null_catalog_name_for_resolved_reference_catalog"); - EntityCacheEntry resolvedCachedCatalog = primaryResolver.getResolvedReferenceCatalog(); - if (resolvedCachedCatalog == null) { + ResolvedPolarisEntity resolvedReferenceCatalog = primaryResolver.getResolvedReferenceCatalog(); + if (resolvedReferenceCatalog == null) { return null; } if (prependRootContainer) { @@ -312,11 +310,9 @@ public PolarisResolvedPathWrapper getResolvedReferenceCatalogEntity( // TODO: Throw appropriate Catalog NOT_FOUND exception before any call to // getResolvedReferenceCatalogEntity(). return new PolarisResolvedPathWrapper( - List.of( - getResolvedRootContainerEntity(), new ResolvedPolarisEntity(resolvedCachedCatalog))); + List.of(getResolvedRootContainerEntity(), resolvedReferenceCatalog)); } else { - return new PolarisResolvedPathWrapper( - List.of(new ResolvedPolarisEntity(resolvedCachedCatalog))); + return new PolarisResolvedPathWrapper(List.of(resolvedReferenceCatalog)); } } @@ -328,7 +324,7 @@ public PolarisEntitySubType getLeafSubType(Object key) { key, pathLookup); int index = pathLookup.get(key); - List resolved = primaryResolver.getResolvedPaths().get(index); + List resolved = primaryResolver.getResolvedPaths().get(index); if (resolved.isEmpty()) { return PolarisEntitySubType.NULL_SUBTYPE; } @@ -357,7 +353,7 @@ public PolarisResolvedPathWrapper getResolvedPath(Object key, boolean prependRoo // Return null for a partially-resolved "optional" path. ResolverPath requestedPath = addedPaths.get(index); - List resolvedPath = primaryResolver.getResolvedPaths().get(index); + List resolvedPath = primaryResolver.getResolvedPaths().get(index); if (requestedPath.isOptional()) { if (resolvedPath.size() != requestedPath.getEntityNames().size()) { return null; @@ -368,8 +364,8 @@ public PolarisResolvedPathWrapper getResolvedPath(Object key, boolean prependRoo if (prependRootContainer) { resolvedEntities.add(getResolvedRootContainerEntity()); } - resolvedEntities.add(new ResolvedPolarisEntity(primaryResolver.getResolvedReferenceCatalog())); - resolvedPath.forEach(cacheEntry -> resolvedEntities.add(new ResolvedPolarisEntity(cacheEntry))); + resolvedEntities.add(primaryResolver.getResolvedReferenceCatalog()); + resolvedPath.forEach(resolvedEntity -> resolvedEntities.add(resolvedEntity)); return new PolarisResolvedPathWrapper(resolvedEntities); } @@ -407,15 +403,15 @@ public PolarisResolvedPathWrapper getResolvedTopLevelEntity( return null; } - EntityCacheEntry resolvedCacheEntry = primaryResolver.getResolvedEntity(entityType, entityName); - if (resolvedCacheEntry == null) { + ResolvedPolarisEntity resolvedEntity = + primaryResolver.getResolvedEntity(entityType, entityName); + if (resolvedEntity == null) { return null; } ResolvedPolarisEntity resolvedRootContainerEntity = getResolvedRootContainerEntity(); return resolvedRootContainerEntity == null - ? new PolarisResolvedPathWrapper(List.of(new ResolvedPolarisEntity(resolvedCacheEntry))) - : new PolarisResolvedPathWrapper( - List.of(resolvedRootContainerEntity, new ResolvedPolarisEntity(resolvedCacheEntry))); + ? new PolarisResolvedPathWrapper(List.of(resolvedEntity)) + : new PolarisResolvedPathWrapper(List.of(resolvedRootContainerEntity, resolvedEntity)); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java index 8141248239..2b85690ba2 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java @@ -40,12 +40,12 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager.ChangeTrackingResult; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; import org.apache.polaris.core.persistence.cache.EntityCacheLookupResult; -import org.apache.polaris.core.persistence.cache.PolarisRemoteCache; -import org.apache.polaris.core.persistence.cache.PolarisRemoteCache.ChangeTrackingResult; /** * REST request resolver, allows to resolve all entities referenced directly or indirectly by in @@ -60,7 +60,7 @@ public class Resolver { private final @Nonnull PolarisDiagnostics diagnostics; // the polaris metastore manager - private final @Nonnull PolarisRemoteCache polarisRemoteCache; + private final @Nonnull PolarisMetaStoreManager polarisMetaStoreManager; // the cache of entities private final @Nonnull EntityCache cache; @@ -81,25 +81,27 @@ public class Resolver { private final List pathsToResolve; // caller principal - private EntityCacheEntry resolvedCallerPrincipal; + private ResolvedPolarisEntity resolvedCallerPrincipal; // all principal roles which have been resolved - private List resolvedCallerPrincipalRoles; + private List resolvedCallerPrincipalRoles; // catalog to use as the reference catalog for role activation - private EntityCacheEntry resolvedReferenceCatalog; + private ResolvedPolarisEntity resolvedReferenceCatalog; // all catalog roles which have been activated - private final Map resolvedCatalogRoles; + private final Map resolvedCatalogRoles; // all resolved paths - private List> resolvedPaths; + private List> resolvedPaths; - // all entities which have been successfully resolved, by name - private final Map resolvedEntriesByName; + // all entities which have been successfully resolved, by name. The entries may or may not + // have come from a cache, but we use the EntityCacheByNameKey anyways as a convenient + // canonical by-name key. + private final Map resolvedEntriesByName; // all entities which have been fully resolved, by id - private final Map resolvedEntriesById; + private final Map resolvedEntriesById; private ResolverStatus resolverStatus; @@ -107,7 +109,7 @@ public class Resolver { * Constructor, effectively starts an entity resolver session * * @param polarisCallContext the polaris call context - * @param polarisRemoteCache meta store manager + * @param polarisMetaStoreManager meta store manager * @param securityContext The {@link AuthenticatedPolarisPrincipal} for the current request * @param cache shared entity cache * @param referenceCatalogName if not null, specifies the name of the reference catalog. The @@ -120,20 +122,21 @@ public class Resolver { */ public Resolver( @Nonnull PolarisCallContext polarisCallContext, - @Nonnull PolarisRemoteCache polarisRemoteCache, + @Nonnull PolarisMetaStoreManager polarisMetaStoreManager, @Nonnull SecurityContext securityContext, @Nonnull EntityCache cache, @Nullable String referenceCatalogName) { this.polarisCallContext = polarisCallContext; this.diagnostics = polarisCallContext.getDiagServices(); - this.polarisRemoteCache = polarisRemoteCache; + this.polarisMetaStoreManager = polarisMetaStoreManager; this.cache = cache; this.securityContext = securityContext; this.referenceCatalogName = referenceCatalogName; // validate inputs this.diagnostics.checkNotNull(polarisCallContext, "unexpected_null_polarisCallContext"); - this.diagnostics.checkNotNull(polarisRemoteCache, "unexpected_null_polarisRemoteCache"); + this.diagnostics.checkNotNull( + polarisMetaStoreManager, "unexpected_null_polarisMetaStoreManager"); this.diagnostics.checkNotNull(cache, "unexpected_null_cache"); this.diagnostics.checkNotNull(securityContext, "security_context_must_be_specified"); this.diagnostics.checkNotNull( @@ -264,7 +267,7 @@ public ResolverStatus resolveAll() { /** * @return the principal we resolved */ - public @Nonnull EntityCacheEntry getResolvedCallerPrincipal() { + public @Nonnull ResolvedPolarisEntity getResolvedCallerPrincipal() { // can only be called if the resolver has been called and was success this.diagnostics.checkNotNull(resolverStatus, "resolver_must_be_called_first"); this.diagnostics.check( @@ -277,7 +280,7 @@ public ResolverStatus resolveAll() { /** * @return all principal roles which were activated. The list can be empty */ - public @Nonnull List getResolvedCallerPrincipalRoles() { + public @Nonnull List getResolvedCallerPrincipalRoles() { // can only be called if the resolver has been called and was success this.diagnostics.checkNotNull(resolverStatus, "resolver_must_be_called_first"); this.diagnostics.check( @@ -291,7 +294,7 @@ public ResolverStatus resolveAll() { * @return the reference catalog which has been resolved. Will be null if null was passed in for * the parameter referenceCatalogName when the Resolver was constructed. */ - public @Nullable EntityCacheEntry getResolvedReferenceCatalog() { + public @Nullable ResolvedPolarisEntity getResolvedReferenceCatalog() { // can only be called if the resolver has been called and was success this.diagnostics.checkNotNull(resolverStatus, "resolver_must_be_called_first"); this.diagnostics.check( @@ -307,7 +310,7 @@ public ResolverStatus resolveAll() { * * @return map of activated catalog roles or null if no referenceCatalogName was specified */ - public @Nullable Map getResolvedCatalogRoles() { + public @Nullable Map getResolvedCatalogRoles() { // can only be called if the resolver has been called and was success this.diagnostics.checkNotNull(resolverStatus, "resolver_must_be_called_first"); this.diagnostics.check( @@ -324,7 +327,7 @@ public ResolverStatus resolveAll() { * * @return single resolved path */ - public @Nonnull List getResolvedPath() { + public @Nonnull List getResolvedPath() { // can only be called if the resolver has been called and was success this.diagnostics.checkNotNull(resolverStatus, "resolver_must_be_called_first"); this.diagnostics.check( @@ -340,7 +343,7 @@ public ResolverStatus resolveAll() { * * @return list of resolved path */ - public @Nonnull List> getResolvedPaths() { + public @Nonnull List> getResolvedPaths() { // can only be called if the resolver has been called and was success this.diagnostics.checkNotNull(resolverStatus, "resolver_must_be_called_first"); this.diagnostics.check( @@ -360,7 +363,7 @@ public ResolverStatus resolveAll() { * @param entityName name of the entity. * @return the entity which has been resolved or null if that entity does not exist */ - public @Nullable EntityCacheEntry getResolvedEntity( + public @Nullable ResolvedPolarisEntity getResolvedEntity( @Nonnull PolarisEntityType entityType, @Nonnull String entityName) { // can only be called if the resolver has been called and was success this.diagnostics.checkNotNull(resolverStatus, "resolver_must_be_called_first"); @@ -400,8 +403,9 @@ private ResolverStatus runResolvePass() { this.resolvedCallerPrincipalRoles.clear(); this.resolvedPaths.clear(); - // all entries we found in the cache but that we need to validate since they might be stale - List toValidate = new ArrayList<>(); + // all entries we found in the cache or resolved hierarchically but that we need to validate + // since they might be stale + List toValidate = new ArrayList<>(); // first resolve the principal and determine the set of activated principal roles ResolverStatus status = this.resolveCallerPrincipalAndPrincipalRoles(toValidate); @@ -428,9 +432,10 @@ private ResolverStatus runResolvePass() { } // all the above resolution was optimistic i.e. when we probe the cache and find an entity, we - // don't validate if this entity has been changed in the backend. So validate now all these - // entities in one single - // go, + // don't validate if this entity has been changed in the backend. Also, hierarchical entities + // were resolved incrementally and may have changed in ways that impact the behavior of + // resolved child entities. So validate now all these entities in one single go, which ensures + // happens-before semantics. boolean validationSuccess = this.bulkValidate(toValidate); if (validationSuccess) { @@ -448,36 +453,37 @@ private void updateResolved() { // if success, we need to get the validated entries // we will resolve those again - this.resolvedCallerPrincipal = this.getResolved(this.resolvedCallerPrincipal); + this.resolvedCallerPrincipal = this.getFreshlyResolved(this.resolvedCallerPrincipal); // update all principal roles with latest if (!this.resolvedCallerPrincipalRoles.isEmpty()) { - List refreshedResolvedCallerPrincipalRoles = + List refreshedResolvedCallerPrincipalRoles = new ArrayList<>(this.resolvedCallerPrincipalRoles.size()); this.resolvedCallerPrincipalRoles.forEach( - ce -> refreshedResolvedCallerPrincipalRoles.add(this.getResolved(ce))); + ce -> refreshedResolvedCallerPrincipalRoles.add(this.getFreshlyResolved(ce))); this.resolvedCallerPrincipalRoles = refreshedResolvedCallerPrincipalRoles; } // update referenced catalog - this.resolvedReferenceCatalog = this.getResolved(this.resolvedReferenceCatalog); + this.resolvedReferenceCatalog = this.getFreshlyResolved(this.resolvedReferenceCatalog); // update all resolved catalog roles if (this.resolvedCatalogRoles != null) { - for (EntityCacheEntry catalogCacheEntry : this.resolvedCatalogRoles.values()) { + for (ResolvedPolarisEntity catalogResolvedEntity : this.resolvedCatalogRoles.values()) { this.resolvedCatalogRoles.put( - catalogCacheEntry.getEntity().getId(), this.getResolved(catalogCacheEntry)); + catalogResolvedEntity.getEntity().getId(), + this.getFreshlyResolved(catalogResolvedEntity)); } } // update all resolved paths if (!this.resolvedPaths.isEmpty()) { - List> refreshedResolvedPaths = + List> refreshedResolvedPaths = new ArrayList<>(this.resolvedPaths.size()); this.resolvedPaths.forEach( rp -> { - List refreshedRp = new ArrayList<>(rp.size()); - rp.forEach(ce -> refreshedRp.add(this.getResolved(ce))); + List refreshedRp = new ArrayList<>(rp.size()); + rp.forEach(ce -> refreshedRp.add(this.getFreshlyResolved(ce))); refreshedResolvedPaths.add(refreshedRp); }); this.resolvedPaths = refreshedResolvedPaths; @@ -485,76 +491,80 @@ private void updateResolved() { } /** - * Get the fully resolved cache entry for the specified cache entry + * Exchange a possibly-stale entity for the latest resolved version of that entity * - * @param cacheEntry input cache entry - * @return the fully resolved cached entry which will often be the same + * @param originalEntity original resolved entity for which to get the latest resolved version + * @return the fully resolved entry which will often be the same */ - private EntityCacheEntry getResolved(EntityCacheEntry cacheEntry) { - final EntityCacheEntry refreshedEntry; - if (cacheEntry == null) { + private ResolvedPolarisEntity getFreshlyResolved(ResolvedPolarisEntity originalEntity) { + final ResolvedPolarisEntity refreshedEntry; + if (originalEntity == null) { refreshedEntry = null; } else { // the latest refreshed entry - refreshedEntry = this.resolvedEntriesById.get(cacheEntry.getEntity().getId()); + refreshedEntry = this.resolvedEntriesById.get(originalEntity.getEntity().getId()); this.diagnostics.checkNotNull( - refreshedEntry, "cache_entry_should_be_resolved", "entity={}", cacheEntry.getEntity()); + refreshedEntry, "_entry_should_be_resolved", "entity={}", originalEntity.getEntity()); } return refreshedEntry; } /** * Bulk validate now the set of entities we didn't validate when we were accessing the entity - * cache + * cache or incrementally resolving * * @param toValidate entities to validate - * @return true if none of the entities in the cache has changed + * @return true if none of the entities has changed */ - private boolean bulkValidate(List toValidate) { + private boolean bulkValidate(List toValidate) { // assume everything is good boolean validationStatus = true; // bulk validate if (!toValidate.isEmpty()) { + // TODO: Provide configurable option to enforce bulk validation of *all* entities in a + // resolution pass, instead of only validating ones on "cache hit"; this would allow the same + // semantics as the transactional validation performed for methods like readEntityByName + // when PolarisMetaStoreManagerImpl uses PolarisEntityResolver in a read transaction. List entityIds = toValidate.stream() .map( - cacheEntry -> + resolvedEntity -> new PolarisEntityId( - cacheEntry.getEntity().getCatalogId(), cacheEntry.getEntity().getId())) + resolvedEntity.getEntity().getCatalogId(), + resolvedEntity.getEntity().getId())) .collect(Collectors.toList()); // now get the current backend versions of all these entities ChangeTrackingResult changeTrackingResult = - this.polarisRemoteCache.loadEntitiesChangeTracking(this.polarisCallContext, entityIds); + this.polarisMetaStoreManager.loadEntitiesChangeTracking( + this.polarisCallContext, entityIds); // refresh any entity which is not fresh. If an entity is missing, reload it - Iterator entityIterator = toValidate.iterator(); + Iterator entityIterator = toValidate.iterator(); Iterator versionIterator = changeTrackingResult.getChangeTrackingVersions().iterator(); // determine the ones we need to reload or refresh and the ones which are up-to-date while (entityIterator.hasNext()) { - // get cache entry and associated versions - EntityCacheEntry cacheEntry = entityIterator.next(); + // get resolved entity and associated versions + ResolvedPolarisEntity resolvedEntity = entityIterator.next(); PolarisChangeTrackingVersions versions = versionIterator.next(); + PolarisBaseEntity entity = resolvedEntity.getEntity(); - // entity we found in the cache - PolarisBaseEntity entity = cacheEntry.getEntity(); - - // refresh cache entry if the entity or grant records version is different - final EntityCacheEntry refreshedCacheEntry; + // refresh the resolved entity if the entity or grant records version is different + final ResolvedPolarisEntity refreshedResolvedEntity; if (versions == null || entity.getEntityVersion() != versions.getEntityVersion() || entity.getGrantRecordsVersion() != versions.getGrantRecordsVersion()) { // if null version we need to invalidate the cached entry since it has probably been // dropped if (versions == null) { - this.cache.removeCacheEntry(cacheEntry); - refreshedCacheEntry = null; + this.cache.removeCacheEntry(resolvedEntity); + refreshedResolvedEntity = null; } else { // refresh that entity. If versions is null, it has been dropped - refreshedCacheEntry = + refreshedResolvedEntity = this.cache.getAndRefreshIfNeeded( this.polarisCallContext, entity, @@ -564,7 +574,7 @@ private boolean bulkValidate(List toValidate) { // get the refreshed entity PolarisBaseEntity refreshedEntity = - (refreshedCacheEntry == null) ? null : refreshedCacheEntry.getEntity(); + (refreshedResolvedEntity == null) ? null : refreshedResolvedEntity.getEntity(); // if the entity has been removed, or its name has changed, or it was re-parented, or it // was dropped, we will have to perform another pass @@ -584,13 +594,13 @@ private boolean bulkValidate(List toValidate) { } } else { // no need to refresh, it is up-to-date - refreshedCacheEntry = cacheEntry; + refreshedResolvedEntity = resolvedEntity; } // if it was found, it has been resolved, so if there is another pass, we will not have to // resolve it again - if (refreshedCacheEntry != null) { - this.addToResolved(refreshedCacheEntry); + if (refreshedResolvedEntity != null) { + this.addToResolved(refreshedResolvedEntity); } } } @@ -602,17 +612,18 @@ private boolean bulkValidate(List toValidate) { /** * Resolve a set of top-level service or catalog entities * - * @param toValidate all entities we have resolved from the cache, hence we will have to verify - * that these entities have not changed in the backend + * @param toValidate all entities we have resolved incrementally, possibly with some entries + * coming from cache, hence we will have to verify that these entities have not changed in the + * backend * @param entitiesToResolve the set of entities to resolve * @return the status of resolution */ private ResolverStatus resolveEntities( - List toValidate, AbstractSet entitiesToResolve) { + List toValidate, AbstractSet entitiesToResolve) { // resolve each for (ResolverEntityName entityName : entitiesToResolve) { // resolve that entity - EntityCacheEntry resolvedEntity = + ResolvedPolarisEntity resolvedEntity = this.resolveByName(toValidate, entityName.getEntityType(), entityName.getEntityName()); // if not found, we can exit unless the entity is optional @@ -629,13 +640,14 @@ private ResolverStatus resolveEntities( /** * Resolve a set of path inside the referenced catalog * - * @param toValidate all entities we have resolved from the cache, hence we will have to verify - * that these entities have not changed in the backend + * @param toValidate all entities we have resolved incrementally, possibly with some entries + * coming from cache, hence we will have to verify that these entities have not changed in the + * backend * @param pathsToResolve the set of paths to resolve * @return the status of resolution */ private ResolverStatus resolvePaths( - List toValidate, List pathsToResolve) { + List toValidate, List pathsToResolve) { // id of the catalog for all these paths final long catalogId = this.resolvedReferenceCatalog.getEntity().getId(); @@ -644,7 +656,7 @@ private ResolverStatus resolvePaths( for (ResolverPath path : pathsToResolve) { // path we are resolving - List resolvedPath = new ArrayList<>(); + List resolvedPath = new ArrayList<>(); // initial parent id is the catalog itself long parentId = catalogId; @@ -660,7 +672,7 @@ private ResolverStatus resolvePaths( pathIt.hasNext() ? PolarisEntityType.NAMESPACE : path.getLastEntityType(); // resolve that entity - EntityCacheEntry segment = + ResolvedPolarisEntity segment = this.resolveByName(toValidate, catalogId, segmentType, parentId, segmentName); // if not found, abort @@ -691,12 +703,13 @@ private ResolverStatus resolvePaths( /** * Resolve the principal and determine which principal roles are activated. Resolved those. * - * @param toValidate all entities we have resolved from the cache, hence we will have to verify - * that these entities have not changed in the backend + * @param toValidate all entities we have resolved incrementally, possibly with some entries + * coming from cache, hence we will have to verify that these entities have not changed in the + * backend * @return the status of resolution */ private ResolverStatus resolveCallerPrincipalAndPrincipalRoles( - List toValidate) { + List toValidate) { // resolve the principal, by name or id this.resolvedCallerPrincipal = @@ -730,8 +743,8 @@ private ResolverStatus resolveCallerPrincipalAndPrincipalRoles( * @param resolvedCallerPrincipal1 * @return the list of resolved principal roles the principal has grants for */ - private List resolveAllPrincipalRoles( - List toValidate, EntityCacheEntry resolvedCallerPrincipal1) { + private List resolveAllPrincipalRoles( + List toValidate, ResolvedPolarisEntity resolvedCallerPrincipal1) { return resolvedCallerPrincipal1.getGrantRecordsAsGrantee().stream() .filter(gr -> gr.getPrivilegeCode() == PolarisPrivilege.PRINCIPAL_ROLE_USAGE.getCode()) .map( @@ -752,8 +765,8 @@ private List resolveAllPrincipalRoles( * @param roleNames * @return the filtered list of resolved principal roles */ - private List resolvePrincipalRolesByName( - List toValidate, Set roleNames) { + private List resolvePrincipalRolesByName( + List toValidate, Set roleNames) { return roleNames.stream() .filter(securityContext::isUserInRole) .map(roleName -> resolveByName(toValidate, PolarisEntityType.PRINCIPAL_ROLE, roleName)) @@ -764,14 +777,15 @@ private List resolvePrincipalRolesByName( * Resolve the reference catalog and determine all activated role. The principal and principal * roles should have already been resolved * - * @param toValidate all entities we have resolved from the cache, hence we will have to verify - * that these entities have not changed in the backend + * @param toValidate all entities we have resolved incrementally, possibly with some entries + * coming from cache, hence we will have to verify that these entities have not changed in the + * backend * @param referenceCatalogName name of the reference catalog to resolve, along with all catalog * roles which are activated * @return the status of resolution */ private ResolverStatus resolveReferenceCatalog( - @Nonnull List toValidate, @Nonnull String referenceCatalogName) { + @Nonnull List toValidate, @Nonnull String referenceCatalogName) { // resolve the catalog this.resolvedReferenceCatalog = this.resolveByName(toValidate, PolarisEntityType.CATALOG, referenceCatalogName); @@ -784,7 +798,7 @@ private ResolverStatus resolveReferenceCatalog( // determine the set of catalog roles which have been activated long catalogId = this.resolvedReferenceCatalog.getEntity().getId(); - for (EntityCacheEntry principalRole : resolvedCallerPrincipalRoles) { + for (ResolvedPolarisEntity principalRole : resolvedCallerPrincipalRoles) { for (PolarisGrantRecord grantRecord : principalRole.getGrantRecordsAsGrantee()) { // the securable is a catalog role belonging to if (grantRecord.getPrivilegeCode() == PolarisPrivilege.CATALOG_ROLE_USAGE.getCode() @@ -795,7 +809,7 @@ private ResolverStatus resolveReferenceCatalog( // skip if it has already been added if (!this.resolvedCatalogRoles.containsKey(catalogRoleId)) { // see if this catalog can be resolved - EntityCacheEntry catalogRole = + ResolvedPolarisEntity catalogRole = this.resolveById( toValidate, PolarisEntityType.CATALOG_ROLE, catalogId, catalogRoleId); @@ -813,23 +827,23 @@ private ResolverStatus resolveReferenceCatalog( } /** - * Add a cache entry to the set of resolved entities + * Add a resolved entity to the current resolution collection's set of resolved entities * - * @param refreshedCacheEntry refreshed cache entry + * @param refreshedResolvedEntity refreshed resolved entity */ - private void addToResolved(EntityCacheEntry refreshedCacheEntry) { + private void addToResolved(ResolvedPolarisEntity refreshedResolvedEntity) { // underlying entity - PolarisBaseEntity entity = refreshedCacheEntry.getEntity(); + PolarisBaseEntity entity = refreshedResolvedEntity.getEntity(); // add it by ID - this.resolvedEntriesById.put(entity.getId(), refreshedCacheEntry); + this.resolvedEntriesById.put(entity.getId(), refreshedResolvedEntity); // in the by name map, only add it if it has not been dropped if (!entity.isDropped()) { this.resolvedEntriesByName.put( new EntityCacheByNameKey( entity.getCatalogId(), entity.getParentId(), entity.getType(), entity.getName()), - refreshedCacheEntry); + refreshedResolvedEntity); } } @@ -869,10 +883,10 @@ private void addEntityByName( * @param toValidate set of entries we will have to validate * @param entityType entity type * @param entityName name of the entity to resolve - * @return cache entry created for that entity + * @return resolved entity */ - private EntityCacheEntry resolveByName( - List toValidate, PolarisEntityType entityType, String entityName) { + private ResolvedPolarisEntity resolveByName( + List toValidate, PolarisEntityType entityType, String entityName) { if (entityType.isTopLevel()) { return this.resolveByName( toValidate, @@ -897,8 +911,8 @@ private EntityCacheEntry resolveByName( * @return the resolve entity. Potentially update the toValidate list if we will have to validate * that this entity is up-to-date */ - private EntityCacheEntry resolveByName( - @Nonnull List toValidate, + private ResolvedPolarisEntity resolveByName( + @Nonnull List toValidate, long catalogId, @Nonnull PolarisEntityType entityType, long parentId, @@ -909,14 +923,14 @@ private EntityCacheEntry resolveByName( new EntityCacheByNameKey(catalogId, parentId, entityType, entityName); // first check if this entity has not yet been resolved - EntityCacheEntry cacheEntry = this.resolvedEntriesByName.get(nameKey); - if (cacheEntry != null) { - return cacheEntry; + ResolvedPolarisEntity resolvedEntity = this.resolvedEntriesByName.get(nameKey); + if (resolvedEntity != null) { + return resolvedEntity; } // then check if it does not exist in the toValidate list. The same entity might be resolved // several times with multi-path resolution - for (EntityCacheEntry ce : toValidate) { + for (ResolvedPolarisEntity ce : toValidate) { PolarisBaseEntity entity = ce.getEntity(); if (entity.getCatalogId() == catalogId && entity.getParentId() == parentId @@ -960,8 +974,8 @@ private EntityCacheEntry resolveByName( * @return the resolve entity. Potentially update the toValidate list if we will have to validate * that this entity is up-to-date */ - private EntityCacheEntry resolveById( - @Nonnull List toValidate, + private ResolvedPolarisEntity resolveById( + @Nonnull List toValidate, @Nonnull PolarisEntityType entityType, long catalogId, long entityId) { diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java index c47367769e..8dc3c84aba 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java @@ -32,7 +32,6 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; import org.apache.polaris.core.persistence.cache.EntityCacheLookupResult; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -141,7 +140,7 @@ void testGetOrLoadEntityByName() { EntityCacheByNameKey N1_name = new EntityCacheByNameKey( catalog.getId(), catalog.getId(), PolarisEntityType.NAMESPACE, "N1"); - EntityCacheEntry cacheEntry = cache.getEntityByName(N1_name); + ResolvedPolarisEntity cacheEntry = cache.getEntityByName(N1_name); Assertions.assertThat(cacheEntry).isNull(); // try to find it in the cache by id. Should not be there, i.e. no cache hit @@ -162,7 +161,7 @@ void testGetOrLoadEntityByName() { Assertions.assertThat(cacheEntry.getGrantRecordsAsSecurable()).isNotNull(); // lookup N1 - EntityCacheEntry N1_entry = cache.getEntityById(N1.getId()); + ResolvedPolarisEntity N1_entry = cache.getEntityById(N1.getId()); Assertions.assertThat(N1_entry).isNotNull(); Assertions.assertThat(N1_entry.getEntity()).isNotNull(); Assertions.assertThat(N1_entry.getGrantRecordsAsSecurable()).isNotNull(); @@ -181,7 +180,7 @@ void testGetOrLoadEntityByName() { new EntityCacheByNameKey(catalog.getId(), N1.getId(), PolarisEntityType.NAMESPACE, "N2"); lookup = cache.getOrLoadEntityByName(callCtx, N2_name); Assertions.assertThat(lookup).isNotNull(); - EntityCacheEntry cacheEntry_N1 = lookup.getCacheEntry(); + ResolvedPolarisEntity cacheEntry_N1 = lookup.getCacheEntry(); Assertions.assertThat(cacheEntry_N1).isNotNull(); Assertions.assertThat(cacheEntry_N1.getEntity()).isNotNull(); Assertions.assertThat(cacheEntry_N1.getGrantRecordsAsSecurable()).isNotNull(); @@ -192,7 +191,7 @@ void testGetOrLoadEntityByName() { catalog.getId(), catalog.getId(), PolarisEntityType.CATALOG_ROLE, "R1"); lookup = cache.getOrLoadEntityByName(callCtx, R1_name); Assertions.assertThat(lookup).isNotNull(); - EntityCacheEntry cacheEntry_R1 = lookup.getCacheEntry(); + ResolvedPolarisEntity cacheEntry_R1 = lookup.getCacheEntry(); Assertions.assertThat(cacheEntry_R1).isNotNull(); Assertions.assertThat(cacheEntry_R1.getEntity()).isNotNull(); Assertions.assertThat(cacheEntry_R1.getGrantRecordsAsSecurable()).isNotNull(); @@ -236,7 +235,7 @@ void testGetOrLoadEntityByName() { new EntityCacheByNameKey(PolarisEntityType.PRINCIPAL_ROLE, "PR1"); lookup = cache.getOrLoadEntityByName(callCtx, PR1_name); Assertions.assertThat(lookup).isNotNull(); - EntityCacheEntry cacheEntry_PR1 = lookup.getCacheEntry(); + ResolvedPolarisEntity cacheEntry_PR1 = lookup.getCacheEntry(); Assertions.assertThat(cacheEntry_PR1).isNotNull(); Assertions.assertThat(cacheEntry_PR1.getEntity()).isNotNull(); Assertions.assertThat(cacheEntry_PR1.getGrantRecordsAsSecurable()).isNotNull(); @@ -299,7 +298,7 @@ void testRefresh() { Assertions.assertThat(T6v1).isNotNull(); // that table is not in the cache - EntityCacheEntry cacheEntry = cache.getEntityById(T6v1.getId()); + ResolvedPolarisEntity cacheEntry = cache.getEntityById(T6v1.getId()); Assertions.assertThat(cacheEntry).isNull(); // now load that table in the cache @@ -433,7 +432,7 @@ void testRenameAndCacheDestinationBeforeLoadingSource() { new EntityCacheByNameKey(N1.getCatalogId(), N1.getId(), PolarisEntityType.TABLE_LIKE, "T4"); lookup = cache.getOrLoadEntityByName(callCtx, T4_name); Assertions.assertThat(lookup).isNotNull(); - EntityCacheEntry cacheEntry_T4 = lookup.getCacheEntry(); + ResolvedPolarisEntity cacheEntry_T4 = lookup.getCacheEntry(); Assertions.assertThat(cacheEntry_T4).isNotNull(); Assertions.assertThat(cacheEntry_T4.getEntity()).isNotNull(); Assertions.assertThat(cacheEntry_T4.getGrantRecordsAsSecurable()).isNotNull(); @@ -448,7 +447,7 @@ void testRenameAndCacheDestinationBeforeLoadingSource() { N1.getCatalogId(), N1.getId(), PolarisEntityType.TABLE_LIKE, "T4_renamed"); lookup = cache.getOrLoadEntityByName(callCtx, T4_renamed); Assertions.assertThat(lookup).isNotNull(); - EntityCacheEntry cacheEntry_T4_renamed = lookup.getCacheEntry(); + ResolvedPolarisEntity cacheEntry_T4_renamed = lookup.getCacheEntry(); Assertions.assertThat(cacheEntry_T4_renamed).isNotNull(); PolarisBaseEntity T4_renamed_entity = cacheEntry_T4_renamed.getEntity(); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java index 9bc6cabfee..5376a9ab77 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java @@ -44,9 +44,8 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager.ResolvedEntityResult; import org.apache.polaris.core.persistence.cache.EntityCache; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; -import org.apache.polaris.core.persistence.cache.PolarisRemoteCache.CachedEntryResult; import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; @@ -245,7 +244,7 @@ void testResolvePath() { Resolver resolver = this.resolveDriver(this.cache, "test", null, List.of(N1, N5_N6_T8, N5_N6_T5, N1_N2), null); // get all the resolved paths - List> resolvedPath = resolver.getResolvedPaths(); + List> resolvedPath = resolver.getResolvedPaths(); Assertions.assertThat(resolvedPath.get(0)).hasSize(1); Assertions.assertThat(resolvedPath.get(1)).hasSize(2); Assertions.assertThat(resolvedPath.get(2)).hasSize(3); @@ -541,7 +540,7 @@ private void resolvePrincipalAndPrincipalRole( this.ensureResolved(resolver.getResolvedCallerPrincipal(), PolarisEntityType.PRINCIPAL, "P1"); // validate that the two principal roles have been activated - List principalRolesResolved = resolver.getResolvedCallerPrincipalRoles(); + List principalRolesResolved = resolver.getResolvedCallerPrincipalRoles(); // expect two principal roles Assertions.assertThat(principalRolesResolved).hasSize(2); @@ -774,7 +773,8 @@ private Resolver resolveDriver( } // validate that the correct set if principal roles have been activated - List principalRolesResolved = resolver.getResolvedCallerPrincipalRoles(); + List principalRolesResolved = + resolver.getResolvedCallerPrincipalRoles(); principalRolesResolved.sort(Comparator.comparing(p -> p.getEntity().getName())); // expect two principal roles if not scoped @@ -795,7 +795,7 @@ private Resolver resolveDriver( Assertions.assertThat(principalRolesResolved).hasSize(expectedSize); // expect either PR1 and PR2 - for (EntityCacheEntry principalRoleResolved : principalRolesResolved) { + for (ResolvedPolarisEntity principalRoleResolved : principalRolesResolved) { Assertions.assertThat(principalRoleResolved).isNotNull(); Assertions.assertThat(principalRoleResolved.getEntity()).isNotNull(); String roleName = principalRoleResolved.getEntity().getName(); @@ -817,14 +817,14 @@ private Resolver resolveDriver( // if a catalog was passed-in, ensure it exists if (catalogName != null) { - EntityCacheEntry catalogEntry = + ResolvedPolarisEntity catalogEntry = resolver.getResolvedEntity(PolarisEntityType.CATALOG, catalogName); Assertions.assertThat(catalogEntry).isNotNull(); this.ensureResolved(catalogEntry, PolarisEntityType.CATALOG, catalogName); // if a catalog role was passed-in, ensure that it was properly resolved if (catalogRoleName != null) { - EntityCacheEntry catalogRoleEntry = + ResolvedPolarisEntity catalogRoleEntry = resolver.getResolvedEntity(PolarisEntityType.CATALOG_ROLE, catalogRoleName); this.ensureResolved( catalogRoleEntry, @@ -834,7 +834,7 @@ private Resolver resolveDriver( } // validate activated catalog roles - Map activatedCatalogs = resolver.getResolvedCatalogRoles(); + Map activatedCatalogs = resolver.getResolvedCatalogRoles(); // if there is an expected set, ensure we have the same set if (expectedActivatedCatalogRoles != null) { @@ -842,7 +842,7 @@ private Resolver resolveDriver( } // process each of those - for (EntityCacheEntry resolvedActivatedCatalogEntry : activatedCatalogs.values()) { + for (ResolvedPolarisEntity resolvedActivatedCatalogEntry : activatedCatalogs.values()) { // must be in the expected list Assertions.assertThat(resolvedActivatedCatalogEntry).isNotNull(); PolarisBaseEntity activatedCatalogRole = resolvedActivatedCatalogEntry.getEntity(); @@ -867,7 +867,7 @@ private Resolver resolveDriver( List allPathsToCheck = (paths == null) ? List.of(path) : paths; // all resolved path - List> allResolvedPaths = resolver.getResolvedPaths(); + List> allResolvedPaths = resolver.getResolvedPaths(); // same size Assertions.assertThat(allResolvedPaths).hasSameSizeAs(allPathsToCheck); @@ -875,7 +875,7 @@ private Resolver resolveDriver( // check that each path was properly resolved int pathCount = 0; Iterator allPathsToCheckIt = allPathsToCheck.iterator(); - for (List resolvedPath : allResolvedPaths) { + for (List resolvedPath : allResolvedPaths) { this.ensurePathResolved( pathCount++, catalogEntry.getEntity(), allPathsToCheckIt.next(), resolvedPath); } @@ -897,7 +897,7 @@ private void ensurePathResolved( int pathCount, PolarisBaseEntity catalog, ResolverPath pathToResolve, - List resolvedPath) { + List resolvedPath) { // ensure same cardinality if (!pathToResolve.isOptional()) { @@ -910,7 +910,7 @@ private void ensurePathResolved( // loop and validate each element for (int index = 0; index < resolvedPath.size(); index++) { - EntityCacheEntry cacheEntry = resolvedPath.get(index); + ResolvedPolarisEntity cacheEntry = resolvedPath.get(index); String entityName = pathToResolve.getEntityNames().get(index); PolarisEntityType entityType = (index == pathToResolve.getEntityNames().size() - 1) @@ -934,7 +934,7 @@ private void ensurePathResolved( * @param entityName entity name */ private void ensureResolved( - EntityCacheEntry cacheEntry, + ResolvedPolarisEntity cacheEntry, List catalogPath, PolarisEntityType entityType, String entityName) { @@ -952,16 +952,16 @@ private void ensureResolved( Assertions.assertThat(refEntity).isNotNull(); // reload the cached entry from the backend - CachedEntryResult refCachedEntry = - this.metaStoreManager.loadCachedEntryById( + ResolvedEntityResult refResolvedEntity = + this.metaStoreManager.loadResolvedEntityById( this.callCtx, refEntity.getCatalogId(), refEntity.getId()); // should exist - Assertions.assertThat(refCachedEntry).isNotNull(); + Assertions.assertThat(refResolvedEntity).isNotNull(); // ensure same entity - refEntity = refCachedEntry.getEntity(); - List refGrantRecords = refCachedEntry.getEntityGrantRecords(); + refEntity = refResolvedEntity.getEntity(); + List refGrantRecords = refResolvedEntity.getEntityGrantRecords(); Assertions.assertThat(refEntity).isNotNull(); Assertions.assertThat(refGrantRecords).isNotNull(); Assertions.assertThat(entity).isEqualTo(refEntity); @@ -989,7 +989,7 @@ private void ensureResolved( * @param entityName entity name */ private void ensureResolved( - EntityCacheEntry cacheEntry, PolarisEntityType entityType, String entityName) { + ResolvedPolarisEntity cacheEntry, PolarisEntityType entityType, String entityName) { this.ensureResolved(cacheEntry, null, entityType, entityName); } } diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java index 11ee4a87f1..0c7c4131da 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java @@ -43,7 +43,7 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; -import org.apache.polaris.core.persistence.cache.PolarisRemoteCache.CachedEntryResult; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager.ResolvedEntityResult; import org.assertj.core.api.Assertions; /** Test the Polaris persistence layer */ @@ -1226,7 +1226,7 @@ private void validateListReturn( * * @param cacheEntry the cached entity to validate */ - private void validateCacheEntryLoad(CachedEntryResult cacheEntry) { + private void validateCacheEntryLoad(ResolvedEntityResult cacheEntry) { // cannot be null Assertions.assertThat(cacheEntry).isNotNull(); @@ -1287,7 +1287,7 @@ private void validateCacheEntryLoad(CachedEntryResult cacheEntry) { * @param cacheEntry the cached entity to validate */ private void validateCacheEntryRefresh( - CachedEntryResult cacheEntry, + ResolvedEntityResult cacheEntry, long catalogId, long entityId, int entityVersion, @@ -1362,8 +1362,8 @@ private PolarisBaseEntity loadCacheEntryByName( @Nonnull String entityName, boolean expectExists) { // load cached entry - CachedEntryResult cacheEntry = - this.polarisMetaStoreManager.loadCachedEntryByName( + ResolvedEntityResult cacheEntry = + this.polarisMetaStoreManager.loadResolvedEntityByName( this.polarisCallContext, entityCatalogId, parentId, entityType, entityName); // if null, validate that indeed the entry does not exist @@ -1408,8 +1408,8 @@ private PolarisBaseEntity loadCacheEntryByName( private PolarisBaseEntity loadCacheEntryById( long entityCatalogId, long entityId, boolean expectExists) { // load cached entry - CachedEntryResult cacheEntry = - this.polarisMetaStoreManager.loadCachedEntryById( + ResolvedEntityResult cacheEntry = + this.polarisMetaStoreManager.loadResolvedEntityById( this.polarisCallContext, entityCatalogId, entityId); // if null, validate that indeed the entry does not exist @@ -1455,8 +1455,8 @@ private void refreshCacheEntry( long entityId, boolean expectExists) { // load cached entry - CachedEntryResult cacheEntry = - this.polarisMetaStoreManager.refreshCachedEntity( + ResolvedEntityResult cacheEntry = + this.polarisMetaStoreManager.refreshResolvedEntity( this.polarisCallContext, entityVersion, entityGrantRecordsVersion, @@ -2087,8 +2087,8 @@ void renameEntity( // the renamed entity PolarisEntity renamedEntityInput = new PolarisEntity(entity); renamedEntityInput.setName(newName); - String updatedInternalPropertiesString = "updatedDataForInternalProperties1234"; - String updatedPropertiesString = "updatedDataForProperties9876"; + String updatedInternalPropertiesString = "{\"key1\": \"updatedDataForInternalProperties1234\"}"; + String updatedPropertiesString = "{\"key1\": \"updatedDataForProperties9876\"}"; // this is to test that properties are also updated during the rename operation renamedEntityInput.setInternalProperties(updatedInternalPropertiesString); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 65f6665862..5ec318693c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -63,7 +63,7 @@ import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; +import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverStatus; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; @@ -590,7 +590,7 @@ public Response getConfig( if (!resolverStatus.getStatus().equals(ResolverStatus.StatusEnum.SUCCESS)) { throw new NotFoundException("Unable to find warehouse %s", warehouse); } - EntityCacheEntry resolvedReferenceCatalog = resolver.getResolvedReferenceCatalog(); + ResolvedPolarisEntity resolvedReferenceCatalog = resolver.getResolvedReferenceCatalog(); Map properties = PolarisEntity.of(resolvedReferenceCatalog.getEntity()).getPropertiesAsMap(); From 04a636e062671522891c72386f8558d6fe9ddc36 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Mon, 24 Feb 2025 04:19:02 +0000 Subject: [PATCH 02/15] Add support for actually giving null EntityCache to Resolver; parameterize ResolverTest --- .../core/persistence/resolver/Resolver.java | 153 +++++++++++++----- .../core/persistence/ResolverTest.java | 55 +++++-- 2 files changed, 148 insertions(+), 60 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java index 2b85690ba2..52e5d9efc3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java @@ -42,6 +42,7 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager.ChangeTrackingResult; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager.ResolvedEntityResult; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; @@ -137,7 +138,6 @@ public Resolver( this.diagnostics.checkNotNull(polarisCallContext, "unexpected_null_polarisCallContext"); this.diagnostics.checkNotNull( polarisMetaStoreManager, "unexpected_null_polarisMetaStoreManager"); - this.diagnostics.checkNotNull(cache, "unexpected_null_cache"); this.diagnostics.checkNotNull(securityContext, "security_context_must_be_specified"); this.diagnostics.checkNotNull( securityContext.getUserPrincipal(), "principal_must_be_specified"); @@ -560,16 +560,41 @@ private boolean bulkValidate(List toValidate) { // if null version we need to invalidate the cached entry since it has probably been // dropped if (versions == null) { - this.cache.removeCacheEntry(resolvedEntity); + if (this.cache != null) { + this.cache.removeCacheEntry(resolvedEntity); + } refreshedResolvedEntity = null; } else { // refresh that entity. If versions is null, it has been dropped - refreshedResolvedEntity = - this.cache.getAndRefreshIfNeeded( - this.polarisCallContext, - entity, - versions.getEntityVersion(), - versions.getGrantRecordsVersion()); + if (this.cache != null) { + refreshedResolvedEntity = + this.cache.getAndRefreshIfNeeded( + this.polarisCallContext, + entity, + versions.getEntityVersion(), + versions.getGrantRecordsVersion()); + } else { + ResolvedEntityResult result = + this.polarisMetaStoreManager.refreshResolvedEntity( + this.polarisCallContext, + entity.getEntityVersion(), + entity.getGrantRecordsVersion(), + entity.getType(), + entity.getCatalogId(), + entity.getId()); + refreshedResolvedEntity = + result.isSuccess() + ? new ResolvedPolarisEntity( + this.polarisCallContext.getDiagServices(), + result.getEntity() != null ? result.getEntity() : entity, + result.getEntityGrantRecords() != null + ? result.getEntityGrantRecords() + : resolvedEntity.getAllGrantRecords(), + result.getEntityGrantRecords() != null + ? result.getGrantRecordsVersion() + : entity.getGrantRecordsVersion()) + : null; + } } // get the refreshed entity @@ -941,27 +966,47 @@ private ResolvedPolarisEntity resolveByName( } // get or load by name - EntityCacheLookupResult lookupResult = - this.cache.getOrLoadEntityByName( - this.polarisCallContext, - new EntityCacheByNameKey(catalogId, parentId, entityType, entityName)); - - // if not found - if (lookupResult == null) { - // not found - return null; - } else if (lookupResult.isCacheHit()) { - // found in the cache, we will have to validate this entity - toValidate.add(lookupResult.getCacheEntry()); + if (this.cache != null) { + EntityCacheLookupResult lookupResult = + this.cache.getOrLoadEntityByName( + this.polarisCallContext, + new EntityCacheByNameKey(catalogId, parentId, entityType, entityName)); + + // if not found + if (lookupResult == null) { + // not found + return null; + } else if (lookupResult.isCacheHit()) { + // found in the cache, we will have to validate this entity + toValidate.add(lookupResult.getCacheEntry()); + } else { + // entry cannot be null + this.diagnostics.checkNotNull(lookupResult.getCacheEntry(), "cache_entry_is_null"); + // if not found in cache, it was loaded from backend, hence it has been resolved + this.addToResolved(lookupResult.getCacheEntry()); + } + + // return the cache entry + return lookupResult.getCacheEntry(); } else { - // entry cannot be null - this.diagnostics.checkNotNull(lookupResult.getCacheEntry(), "cache_entry_is_null"); - // if not found in cache, it was loaded from backend, hence it has been resolved - this.addToResolved(lookupResult.getCacheEntry()); - } + // If no cache, load directly from metastore manager. + ResolvedEntityResult result = + this.polarisMetaStoreManager.loadResolvedEntityByName( + this.polarisCallContext, catalogId, parentId, entityType, entityName); + if (!result.isSuccess()) { + // not found + return null; + } - // return the cache entry - return lookupResult.getCacheEntry(); + resolvedEntity = + new ResolvedPolarisEntity( + this.polarisCallContext.getDiagServices(), + result.getEntity(), + result.getEntityGrantRecords(), + result.getGrantRecordsVersion()); + this.addToResolved(resolvedEntity); + return resolvedEntity; + } } /** @@ -979,25 +1024,45 @@ private ResolvedPolarisEntity resolveById( @Nonnull PolarisEntityType entityType, long catalogId, long entityId) { - // get or load by name - EntityCacheLookupResult lookupResult = - this.cache.getOrLoadEntityById(this.polarisCallContext, catalogId, entityId); - - // if not found, return null - if (lookupResult == null) { - return null; - } else if (lookupResult.isCacheHit()) { - // found in the cache, we will have to validate this entity - toValidate.add(lookupResult.getCacheEntry()); + if (this.cache != null) { + // get or load by name + EntityCacheLookupResult lookupResult = + this.cache.getOrLoadEntityById(this.polarisCallContext, catalogId, entityId); + + // if not found, return null + if (lookupResult == null) { + return null; + } else if (lookupResult.isCacheHit()) { + // found in the cache, we will have to validate this entity + toValidate.add(lookupResult.getCacheEntry()); + } else { + // entry cannot be null + this.diagnostics.checkNotNull(lookupResult.getCacheEntry(), "cache_entry_is_null"); + + // if not found in cache, it was loaded from backend, hence it has been resolved + this.addToResolved(lookupResult.getCacheEntry()); + } + + // return the cache entry + return lookupResult.getCacheEntry(); } else { - // entry cannot be null - this.diagnostics.checkNotNull(lookupResult.getCacheEntry(), "cache_entry_is_null"); + // If no cache, load directly from metastore manager. + ResolvedEntityResult result = + polarisMetaStoreManager.loadResolvedEntityById( + this.polarisCallContext, catalogId, entityId); + if (!result.isSuccess()) { + // not found + return null; + } - // if not found in cache, it was loaded from backend, hence it has been resolved - this.addToResolved(lookupResult.getCacheEntry()); + ResolvedPolarisEntity resolvedEntity = + new ResolvedPolarisEntity( + this.polarisCallContext.getDiagServices(), + result.getEntity(), + result.getEntityGrantRecords(), + result.getGrantRecordsVersion()); + this.addToResolved(resolvedEntity); + return resolvedEntity; } - - // return the cache entry - return lookupResult.getCacheEntry(); } } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java index 5376a9ab77..5270a90e22 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java @@ -50,7 +50,8 @@ import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; public class ResolverTest { @@ -79,6 +80,12 @@ public class ResolverTest { // cache we are using private EntityCache cache; + // whenever constructing a new Resolver instance, if false, disable cache for that Resolver + // instance by giving it a null cache regardless of the current state of the test-level + // cache instance; use a boolean for this instead of just modifying the test member 'cache' + // so that we can potentially alternate between using cache and not using cache + private boolean shouldUseCache; + /** * Initialize and create the test metadata * @@ -117,8 +124,10 @@ public ResolverTest() { } /** This test resolver for a create-principal scenario */ - @Test - void testResolvePrincipal() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testResolvePrincipal(boolean useCache) { + this.shouldUseCache = useCache; // resolve a principal which does not exist, but make it optional so will succeed this.resolveDriver(null, null, "P3", true, null, null); @@ -144,8 +153,10 @@ void testResolvePrincipal() { } /** Test that we can specify a subset of principal role names */ - @Test - void testScopedPrincipalRole() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testScopedPrincipalRole(boolean useCache) { + this.shouldUseCache = useCache; // start without a scope this.resolveDriver(null, null, "P2", false, "PR1", null); @@ -163,8 +174,10 @@ void testScopedPrincipalRole() { * Test that the set of catalog roles being activated is correctly inferred, based of a set of * principal roles */ - @Test - void testCatalogRolesActivation() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testCatalogRolesActivation(boolean useCache) { + this.shouldUseCache = useCache; // start simple, with both PR1 and PR2, you get R1 and R2 this.resolveDriver(null, Set.of("PR1", "PR2"), "test", Set.of("R1", "R2")); @@ -180,8 +193,11 @@ void testCatalogRolesActivation() { } /** Test that paths, one or more, are properly resolved */ - @Test - void testResolvePath() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testResolvePath(boolean useCache) { + this.shouldUseCache = useCache; + // N1 which exists ResolverPath N1 = new ResolverPath(List.of("N1"), PolarisEntityType.NAMESPACE); this.resolveDriver(null, "test", N1, null, null); @@ -255,8 +271,10 @@ void testResolvePath() { * Ensure that if data changes while entities are cached, we will always resolve to the latest * version */ - @Test - void testConsistency() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testConsistency(boolean useCache) { + this.shouldUseCache = useCache; // resolve principal "P2" this.resolveDriver(null, null, "P2", false, null, null); @@ -315,8 +333,11 @@ void testConsistency() { } /** Check resolve paths when cache is inconsistent */ - @Test - void testPathConsistency() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testPathConsistency(boolean useCache) { + this.shouldUseCache = useCache; + // resolve few paths path ResolverPath N1_PATH = new ResolverPath(List.of("N1"), PolarisEntityType.NAMESPACE); this.resolveDriver(null, "test", N1_PATH, null, null); @@ -362,8 +383,10 @@ void testPathConsistency() { } /** Resolve catalog roles */ - @Test - void testResolveCatalogRole() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testResolveCatalogRole(boolean useCache) { + this.shouldUseCache = useCache; // resolve catalog role this.resolveDriver(null, "test", "R1", null); @@ -492,7 +515,7 @@ public String getAuthenticationScheme() { return ""; } }, - this.cache, + this.shouldUseCache ? this.cache : null, referenceCatalogName); } From 31a837be25b18e783d9698a01923f84076013570 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Mon, 24 Feb 2025 05:29:05 +0000 Subject: [PATCH 03/15] Remove PolarisMetaStoreSession from FileIOFactory/FileIOUtil in favor of CallContext This appeared to be some leaky divergence that occurred after CallContext had been removed, but PolarisMetaStoreSession really is only a low-level implementation detail that should never be handled by BasePolarisCatalog/FileIOFactory. This plumbs CallContext explicitly into the FileIOFactory and FileIOUtil methods and thus removes a large source of CallContext.getCurrentContext calls; now the threadlocal doesn't have to be set at all in BasePolarisCatalogTest. --- .../catalog/BasePolarisCatalogTest.java | 21 +++++++------------ .../ManifestFileCleanupTaskHandlerTest.java | 2 +- .../task/TableCleanupTaskHandlerTest.java | 2 +- .../service/catalog/BasePolarisCatalog.java | 13 +++--------- .../catalog/io/DefaultFileIOFactory.java | 7 ++++--- .../service/catalog/io/FileIOFactory.java | 6 +++--- .../service/catalog/io/FileIOUtil.java | 6 +----- .../io/WasbTranslatingFileIOFactory.java | 6 +++--- .../task/ManifestFileCleanupTaskHandler.java | 2 +- .../service/task/TableCleanupTaskHandler.java | 2 +- .../service/task/TaskFileIOSupplier.java | 8 +++---- .../service/catalog/io/FileIOFactoryTest.java | 2 +- .../catalog/io/MeasuredFileIOFactory.java | 6 +++--- 13 files changed, 33 insertions(+), 50 deletions(-) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 606f70afb0..370ede6623 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -161,6 +161,7 @@ public Map getConfigOverrides() { @Inject PolarisDiagnostics diagServices; private BasePolarisCatalog catalog; + private CallContext callContext; private AwsStorageConfigInfo storageConfigModel; private StsClient stsClient; private String realmName; @@ -199,8 +200,7 @@ public void before(TestInfo testInfo) { new PolarisEntityManager( metaStoreManager, new StorageCredentialCache(), new EntityCache(metaStoreManager)); - CallContext callContext = CallContext.of(realmContext, polarisContext); - CallContext.setCurrentContext(callContext); + callContext = CallContext.of(realmContext, polarisContext); PrincipalEntity rootEntity = new PrincipalEntity( @@ -527,7 +527,7 @@ public void testValidateNotificationFailToCreateFileIO() { final String tableMetadataLocation = tableLocation + "metadata/"; PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( - CallContext.getCurrentContext(), entityManager, securityContext, catalog().name()); + callContext, entityManager, securityContext, catalog().name()); FileIOFactory fileIOFactory = spy( new DefaultFileIOFactory( @@ -538,7 +538,7 @@ public void testValidateNotificationFailToCreateFileIO() { new BasePolarisCatalog( entityManager, metaStoreManager, - CallContext.getCurrentContext(), + callContext, passthroughView, securityContext, Mockito.mock(TaskExecutor.class), @@ -854,7 +854,6 @@ public void testUpdateNotificationCreateTableWithLocalFilePrefix() { .setName(catalogWithoutStorage) .build()); - CallContext callContext = CallContext.getCurrentContext(); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, catalogWithoutStorage); @@ -919,7 +918,6 @@ public void testUpdateNotificationCreateTableWithHttpPrefix() { .setName(catalogName) .build()); - CallContext callContext = CallContext.getCurrentContext(); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, catalogName); @@ -1434,7 +1432,7 @@ public void testDropTableWithPurge() { new RealmEntityManagerFactory(metaStoreManagerFactory), metaStoreManagerFactory, configurationStore)) - .apply(taskEntity, () -> realmName); + .apply(taskEntity, callContext); Assertions.assertThat(fileIO).isNotNull().isInstanceOf(InMemoryFileIO.class); } @@ -1461,8 +1459,6 @@ public void testDropTableWithPurgeDisabled() { .addProperty(PolarisConfiguration.DROP_WITH_PURGE_ENABLED.catalogConfig(), "false") .setStorageConfigurationInfo(noPurgeStorageConfigModel, storageLocation) .build()); - RealmContext realmContext = () -> "realm"; - CallContext callContext = CallContext.of(realmContext, polarisContext); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, noPurgeCatalogName); @@ -1542,9 +1538,6 @@ public void testRetriableException() { @Test public void testFileIOWrapper() { - RealmContext realmContext = () -> "realm"; - CallContext callContext = CallContext.of(realmContext, polarisContext); - CallContext.setCurrentContext(callContext); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, CATALOG_NAME); @@ -1600,7 +1593,7 @@ public void testFileIOWrapper() { new FileIOFactory() { @Override public FileIO loadFileIO( - @NotNull RealmContext realmContext, + @NotNull CallContext callContext, @NotNull String ioImplClassName, @NotNull Map properties, @NotNull TableIdentifier identifier, @@ -1608,7 +1601,7 @@ public FileIO loadFileIO( @NotNull Set storageActions, @NotNull PolarisResolvedPathWrapper resolvedEntityPath) { return measured.loadFileIO( - realmContext, + callContext, "org.apache.iceberg.inmemory.InMemoryFileIO", Map.of(), TABLE, diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java index 16d5222769..b6c03ef8ca 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/ManifestFileCleanupTaskHandlerTest.java @@ -76,7 +76,7 @@ private TaskFileIOSupplier buildTaskFileIOSupplier(FileIO fileIO) { new FileIOFactory() { @Override public FileIO loadFileIO( - @NotNull RealmContext realmContext, + @NotNull CallContext callContext, @NotNull String ioImplClassName, @NotNull Map properties, @NotNull TableIdentifier identifier, diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java index 9680bd1a2c..e107eb830e 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java @@ -75,7 +75,7 @@ private TaskFileIOSupplier buildTaskFileIOSupplier(FileIO fileIO) { new FileIOFactory() { @Override public FileIO loadFileIO( - @Nonnull RealmContext realmContext, + @Nonnull CallContext callContext, @Nonnull String ioImplClassName, @Nonnull Map properties, @Nonnull TableIdentifier identifier, diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index dbee4ca34e..c42bb7fd61 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -836,10 +836,9 @@ public Map getCredentialConfig( return Map.of(); } return FileIOUtil.refreshCredentials( - callContext.getRealmContext(), + callContext, entityManager, getCredentialVendor(), - callContext.getPolarisCallContext().getMetaStore(), callContext.getPolarisCallContext().getConfigurationStore(), tableIdentifier, getLocationsAllowedToBeAccessed(tableMetadata), @@ -1614,7 +1613,7 @@ private FileIO loadFileIOForTableLike( // Reload fileIO based on table specific context FileIO fileIO = fileIOFactory.loadFileIO( - callContext.getRealmContext(), + callContext, ioImplClassName, tableProperties, identifier, @@ -2077,13 +2076,7 @@ private FileIO loadFileIO(String ioImpl, Map properties) { new PolarisResolvedPathWrapper(List.of(resolvedCatalogEntity)); Set storageActions = Set.of(PolarisStorageActions.ALL); return fileIOFactory.loadFileIO( - callContext.getRealmContext(), - ioImpl, - properties, - identifier, - locations, - storageActions, - resolvedPath); + callContext, ioImpl, properties, identifier, locations, storageActions, resolvedPath); } private void blockedUserSpecifiedWriteLocation(Map properties) { diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index 0d8a65569c..2c9b928c94 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -32,6 +32,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.io.FileIO; import org.apache.polaris.core.PolarisConfigurationStore; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; @@ -70,13 +71,14 @@ public DefaultFileIOFactory( @Override public FileIO loadFileIO( - @Nonnull RealmContext realmContext, + @Nonnull CallContext callContext, @Nonnull String ioImplClassName, @Nonnull Map properties, @Nonnull TableIdentifier identifier, @Nonnull Set tableLocations, @Nonnull Set storageActions, @Nonnull PolarisResolvedPathWrapper resolvedEntityPath) { + RealmContext realmContext = callContext.getRealmContext(); PolarisEntityManager entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); PolarisCredentialVendor credentialVendor = @@ -93,10 +95,9 @@ public FileIO loadFileIO( .map( storageInfo -> FileIOUtil.refreshCredentials( - realmContext, + callContext, entityManager, credentialVendor, - metaStoreSession, configurationStore, identifier, tableLocations, diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java index 451aaf716c..f3e0d6b98b 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java @@ -24,7 +24,7 @@ import java.util.Set; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.io.FileIO; -import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -41,7 +41,7 @@ public interface FileIOFactory { *

This method may obtain subscoped credentials to restrict the FileIO's permissions, ensuring * secure and limited access to the table's data and locations. * - * @param realmContext the realm for which the FileIO is being loaded. + * @param callContext the call for which the FileIO is being loaded. * @param ioImplClassName the class name of the FileIO implementation to load. * @param properties configuration properties for the FileIO. * @param identifier the table identifier. @@ -51,7 +51,7 @@ public interface FileIOFactory { * @return a configured FileIO instance. */ FileIO loadFileIO( - @Nonnull RealmContext realmContext, + @Nonnull CallContext callContext, @Nonnull String ioImplClassName, @Nonnull Map properties, @Nonnull TableIdentifier identifier, diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java index 59242fca6c..e0bed634ff 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java @@ -25,11 +25,9 @@ import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisCredentialVendor; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -78,16 +76,14 @@ public static Optional findStorageInfoFromHierarchy( * */ public static Map refreshCredentials( - RealmContext realmContext, + CallContext callContext, PolarisEntityManager entityManager, PolarisCredentialVendor credentialVendor, - PolarisMetaStoreSession metaStoreSession, PolarisConfigurationStore configurationStore, TableIdentifier tableIdentifier, Set tableLocations, Set storageActions, PolarisEntity entity) { - CallContext callContext = CallContext.getCurrentContext(); boolean skipCredentialSubscopingIndirection = configurationStore.getConfiguration( diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java index c98dd4e278..3bb365368d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java @@ -27,7 +27,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.io.FileIO; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -52,7 +52,7 @@ public WasbTranslatingFileIOFactory( @Override public FileIO loadFileIO( - @Nonnull RealmContext realmContext, + @Nonnull CallContext callContext, @Nonnull String ioImplClassName, @Nonnull Map properties, @Nonnull TableIdentifier identifier, @@ -61,7 +61,7 @@ public FileIO loadFileIO( @Nonnull PolarisResolvedPathWrapper resolvedEntityPath) { return new WasbTranslatingFileIO( defaultFileIOFactory.loadFileIO( - realmContext, + callContext, ioImplClassName, properties, identifier, diff --git a/service/common/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java b/service/common/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java index bf6f084a01..9179e0f336 100644 --- a/service/common/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/task/ManifestFileCleanupTaskHandler.java @@ -73,7 +73,7 @@ public boolean canHandleTask(TaskEntity task) { public boolean handleTask(TaskEntity task, CallContext callContext) { ManifestCleanupTask cleanupTask = task.readData(ManifestCleanupTask.class); TableIdentifier tableId = cleanupTask.getTableId(); - try (FileIO authorizedFileIO = fileIOSupplier.apply(task, callContext.getRealmContext())) { + try (FileIO authorizedFileIO = fileIOSupplier.apply(task, callContext)) { if (task.getTaskType() == AsyncTaskType.MANIFEST_FILE_CLEANUP) { ManifestFile manifestFile = decodeManifestData(cleanupTask.getManifestFileData()); return cleanUpManifestFile(manifestFile, authorizedFileIO, tableId); diff --git a/service/common/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java b/service/common/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java index 1f37c58062..3e5dd23f58 100644 --- a/service/common/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java @@ -89,7 +89,7 @@ public boolean handleTask(TaskEntity cleanupTask, CallContext callContext) { // It's likely the cleanupTask has already been completed, but wasn't dropped successfully. // Log a // warning and move on - try (FileIO fileIO = fileIOSupplier.apply(cleanupTask, callContext.getRealmContext())) { + try (FileIO fileIO = fileIOSupplier.apply(cleanupTask, callContext)) { if (!TaskUtils.exists(tableEntity.getMetadataLocation(), fileIO)) { LOGGER .atWarn() diff --git a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java index 7ce7b52906..679d48060b 100644 --- a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java +++ b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java @@ -28,7 +28,7 @@ import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.io.FileIO; -import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisTaskConstants; import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.entity.TaskEntity; @@ -38,7 +38,7 @@ import org.apache.polaris.service.catalog.io.FileIOFactory; @ApplicationScoped -public class TaskFileIOSupplier implements BiFunction { +public class TaskFileIOSupplier implements BiFunction { private final FileIOFactory fileIOFactory; @Inject @@ -47,7 +47,7 @@ public TaskFileIOSupplier(FileIOFactory fileIOFactory) { } @Override - public FileIO apply(TaskEntity task, RealmContext realmContext) { + public FileIO apply(TaskEntity task, CallContext callContext) { Map internalProperties = task.getInternalPropertiesAsMap(); Map properties = new HashMap<>(internalProperties); @@ -65,6 +65,6 @@ public FileIO apply(TaskEntity task, RealmContext realmContext) { CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"); return fileIOFactory.loadFileIO( - realmContext, ioImpl, properties, identifier, locations, storageActions, resolvedPath); + callContext, ioImpl, properties, identifier, locations, storageActions, resolvedPath); } } diff --git a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java index b3eb25eb0a..6c808e77e9 100644 --- a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java @@ -189,7 +189,7 @@ public void testLoadFileIOForCleanupTask() { Assertions.assertThat(tasks).hasSize(1); TaskEntity taskEntity = TaskEntity.of(tasks.get(0)); FileIO fileIO = - new TaskFileIOSupplier(testServices.fileIOFactory()).apply(taskEntity, realmContext); + new TaskFileIOSupplier(testServices.fileIOFactory()).apply(taskEntity, callContext); Assertions.assertThat(fileIO).isNotNull().isInstanceOf(InMemoryFileIO.class); // 1. BasePolarisCatalog:doCommit: for writing the table during the creation diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java b/service/common/src/testFixtures/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java index 943967c9c9..3e831361ff 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/catalog/io/MeasuredFileIOFactory.java @@ -30,7 +30,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.io.FileIO; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -64,7 +64,7 @@ public MeasuredFileIOFactory( @Override public FileIO loadFileIO( - @Nonnull RealmContext realmContext, + @Nonnull CallContext callContext, @Nonnull String ioImplClassName, @Nonnull Map properties, @Nonnull TableIdentifier identifier, @@ -79,7 +79,7 @@ public FileIO loadFileIO( MeasuredFileIO wrapped = new MeasuredFileIO( defaultFileIOFactory.loadFileIO( - realmContext, + callContext, ioImplClassName, properties, identifier, From 7d8942f8e30769be920778ab60496ebd6eeb4869 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Tue, 25 Feb 2025 03:15:50 +0000 Subject: [PATCH 04/15] Restructure persistence class hierarchy and remove vestigial interfaces Extract a basic abstract base class BaseMetaStoreManager which can hold shared logic between implementations of PolarisMetaStoreManager --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 4 +- .../persistence/BaseMetaStoreManager.java | 48 +++++++++++++++++++ .../PolarisMetaStoreManagerImpl.java | 26 ++-------- .../PolarisTreeMapMetaStoreSessionImpl.java | 2 +- .../cache/StorageCredentialCacheTest.java | 3 +- .../catalog/io/DefaultFileIOFactory.java | 3 -- .../context/DefaultCallContextResolver.java | 4 ++ 7 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/persistence/BaseMetaStoreManager.java diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 7950075c94..04787d5125 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -49,7 +49,7 @@ import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.exceptions.AlreadyExistsException; -import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl; +import org.apache.polaris.core.persistence.BaseMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; import org.apache.polaris.core.persistence.RetryOnConcurrencyException; @@ -674,7 +674,7 @@ PolarisStorageIntegration createStorageIntegration( PolarisStorageIntegration loadPolarisStorageIntegration( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { PolarisStorageConfigurationInfo storageConfig = - PolarisMetaStoreManagerImpl.readStorageConfiguration(callCtx, entity); + BaseMetaStoreManager.extractStorageConfiguration(callCtx, entity); return storageIntegrationProvider.getStorageIntegrationForConfig(storageConfig); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BaseMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BaseMetaStoreManager.java new file mode 100644 index 0000000000..c27ee1b377 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BaseMetaStoreManager.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import java.util.Map; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; + +public abstract class BaseMetaStoreManager implements PolarisMetaStoreManager { + public static PolarisStorageConfigurationInfo extractStorageConfiguration( + @Nonnull PolarisCallContext callCtx, PolarisBaseEntity reloadedEntity) { + Map propMap = + PolarisObjectMapperUtil.deserializeProperties( + callCtx, reloadedEntity.getInternalProperties()); + String storageConfigInfoStr = + propMap.get(PolarisEntityConstants.getStorageConfigInfoPropertyName()); + + callCtx + .getDiagServices() + .check( + storageConfigInfoStr != null, + "missing_storage_configuration_info", + "catalogId={}, entityId={}", + reloadedEntity.getCatalogId(), + reloadedEntity.getId()); + return PolarisStorageConfigurationInfo.deserialize( + callCtx.getDiagServices(), storageConfigInfoStr); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 98fd7789ca..6214d0f05f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -63,7 +63,7 @@ * and retrieve all Polaris metadata */ @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE") -public class PolarisMetaStoreManagerImpl implements PolarisMetaStoreManager { +public class PolarisMetaStoreManagerImpl extends BaseMetaStoreManager { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisMetaStoreManagerImpl.class); /** mapper, allows to serialize/deserialize properties to/from JSON */ @@ -2107,7 +2107,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( entityId); PolarisStorageConfigurationInfo storageConfigurationInfo = - readStorageConfiguration(callCtx, reloadedEntity.getEntity()); + BaseMetaStoreManager.extractStorageConfiguration(callCtx, reloadedEntity.getEntity()); try { EnumMap creds = storageIntegration.getSubscopedCreds( @@ -2159,7 +2159,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( // validate access PolarisStorageConfigurationInfo storageConfigurationInfo = - readStorageConfiguration(callCtx, reloadedEntity.getEntity()); + BaseMetaStoreManager.extractStorageConfiguration(callCtx, reloadedEntity.getEntity()); Map validateLocationAccess = storageIntegration .validateAccessToLocations(storageConfigurationInfo, actions, locations) @@ -2174,26 +2174,6 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( return new ValidateAccessResult(validateLocationAccess); } - public static PolarisStorageConfigurationInfo readStorageConfiguration( - @Nonnull PolarisCallContext callCtx, PolarisBaseEntity reloadedEntity) { - Map propMap = - PolarisObjectMapperUtil.deserializeProperties( - callCtx, reloadedEntity.getInternalProperties()); - String storageConfigInfoStr = - propMap.get(PolarisEntityConstants.getStorageConfigInfoPropertyName()); - - callCtx - .getDiagServices() - .check( - storageConfigInfoStr != null, - "missing_storage_configuration_info", - "catalogId={}, entityId={}", - reloadedEntity.getCatalogId(), - reloadedEntity.getId()); - return PolarisStorageConfigurationInfo.deserialize( - callCtx.getDiagServices(), storageConfigInfoStr); - } - /** * Get the internal property map for an entity * diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index bea2054156..6516b7b7fa 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -565,7 +565,7 @@ PolarisStorageIntegration createStorageIntegration( PolarisStorageIntegration loadPolarisStorageIntegration( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { PolarisStorageConfigurationInfo storageConfig = - PolarisMetaStoreManagerImpl.readStorageConfiguration(callCtx, entity); + BaseMetaStoreManager.extractStorageConfiguration(callCtx, entity); return storageIntegrationProvider.getStorageIntegrationForConfig(storageConfig); } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index d19bc41c8b..a4f7c6936d 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -39,7 +39,6 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.BaseResult; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; @@ -70,7 +69,7 @@ public StorageCredentialCacheTest() { PolarisMetaStoreSession metaStore = new PolarisTreeMapMetaStoreSessionImpl(store, Mockito.mock(), RANDOM_SECRETS); callCtx = new PolarisCallContext(metaStore, diagServices); - metaStoreManager = Mockito.mock(PolarisMetaStoreManagerImpl.class); + metaStoreManager = Mockito.mock(PolarisMetaStoreManager.class); storageCredentialCache = new StorageCredentialCache(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index 2c9b928c94..48d9a280ab 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -37,7 +37,6 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisCredentialVendor; import org.apache.polaris.core.storage.PolarisStorageActions; @@ -83,8 +82,6 @@ public FileIO loadFileIO( realmEntityManagerFactory.getOrCreateEntityManager(realmContext); PolarisCredentialVendor credentialVendor = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); - PolarisMetaStoreSession metaStoreSession = - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); // Get subcoped creds properties = new HashMap<>(properties); diff --git a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java index ff39497247..985e57d84f 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java @@ -60,6 +60,10 @@ public CallContext resolveCallContext( .addKeyValue("headers", headers) .log("Resolving CallContext"); + // TODO: Once we have non-transactional-database persistence stores, this should be + // pushed down for the metaStoreManagerFactory to inject Transactional-DB specific things + // (including the MetaStoreSession" into the PolarisCallContext. The non-transactional + // factories would then inject something else instead if needed. PolarisMetaStoreSession metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); PolarisCallContext polarisContext = From 54b762e985bc805200da0317de50aad51747279c Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Tue, 25 Feb 2025 03:44:40 +0000 Subject: [PATCH 05/15] Remove all "entitiesDropped" members; these were vestigial from trying to implement UNDROP but for now are entirely unused. We can reintroduce it with a better design against multiple backends when/if we want to implement UNDROP. --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 16 -- .../eclipselink/PolarisEclipseLinkStore.java | 38 ---- .../jpa/models/ModelEntityDropped.java | 164 ------------------ .../PolarisMetaStoreManagerImpl.java | 110 +++++------- .../persistence/PolarisMetaStoreSession.java | 20 --- .../PolarisTreeMapMetaStoreSessionImpl.java | 18 -- .../core/persistence/PolarisTreeMapStore.java | 49 +----- 7 files changed, 46 insertions(+), 369 deletions(-) delete mode 100644 extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 04787d5125..621993e7fb 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -271,14 +271,6 @@ public void writeToEntitiesActive( this.store.writeToEntitiesActive(localSession.get(), entity); } - /** {@inheritDoc} */ - @Override - public void writeToEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // write it - this.store.writeToEntitiesDropped(localSession.get(), entity); - } - /** {@inheritDoc} */ @Override public void writeToEntitiesChangeTracking( @@ -312,14 +304,6 @@ public void deleteFromEntitiesActive( this.store.deleteFromEntitiesActive(localSession.get(), new PolarisEntitiesActiveKey(entity)); } - /** {@inheritDoc} */ - @Override - public void deleteFromEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // delete it - this.store.deleteFromEntitiesDropped(localSession.get(), entity.getCatalogId(), entity.getId()); - } - /** * {@inheritDoc} * diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index 8216bef91f..59ac63f690 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -38,7 +38,6 @@ import org.apache.polaris.jpa.models.ModelEntity; import org.apache.polaris.jpa.models.ModelEntityActive; import org.apache.polaris.jpa.models.ModelEntityChangeTracking; -import org.apache.polaris.jpa.models.ModelEntityDropped; import org.apache.polaris.jpa.models.ModelGrantRecord; import org.apache.polaris.jpa.models.ModelPrincipalSecrets; import org.slf4j.Logger; @@ -104,17 +103,6 @@ void writeToEntitiesActive(EntityManager session, PolarisBaseEntity entity) { } } - void writeToEntitiesDropped(EntityManager session, PolarisBaseEntity entity) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityDropped entityDropped = - lookupEntityDropped(session, entity.getCatalogId(), entity.getId()); - if (entityDropped == null) { - session.persist(ModelEntityDropped.fromEntity(entity)); - } - } - void writeToEntitiesChangeTracking(EntityManager session, PolarisBaseEntity entity) { diagnosticServices.check(session != null, "session_is_null"); checkInitialized(); @@ -158,16 +146,6 @@ void deleteFromEntitiesActive(EntityManager session, PolarisEntitiesActiveKey ke session.remove(entity); } - void deleteFromEntitiesDropped(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - ModelEntityDropped entity = lookupEntityDropped(session, catalogId, entityId); - diagnosticServices.check(entity != null, "dropped_entity_not_found"); - - session.remove(entity); - } - void deleteFromEntitiesChangeTracking(EntityManager session, PolarisEntityCore entity) { diagnosticServices.check(session != null, "session_is_null"); checkInitialized(); @@ -216,7 +194,6 @@ void deleteAll(EntityManager session) { session.createQuery("DELETE from ModelEntity").executeUpdate(); session.createQuery("DELETE from ModelEntityActive").executeUpdate(); - session.createQuery("DELETE from ModelEntityDropped").executeUpdate(); session.createQuery("DELETE from ModelEntityChangeTracking").executeUpdate(); session.createQuery("DELETE from ModelGrantRecord").executeUpdate(); session.createQuery("DELETE from ModelPrincipalSecrets").executeUpdate(); @@ -319,21 +296,6 @@ List lookupFullEntitiesActive( return query.getResultList(); } - ModelEntityDropped lookupEntityDropped(EntityManager session, long catalogId, long entityId) { - diagnosticServices.check(session != null, "session_is_null"); - checkInitialized(); - - return session - .createQuery( - "SELECT m from ModelEntityDropped m where m.catalogId=:catalogId and m.id=:id", - ModelEntityDropped.class) - .setParameter("catalogId", catalogId) - .setParameter("id", entityId) - .getResultStream() - .findFirst() - .orElse(null); - } - ModelEntityChangeTracking lookupEntityChangeTracking( EntityManager session, long catalogId, long entityId) { diagnosticServices.check(session != null, "session_is_null"); diff --git a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java b/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java deleted file mode 100644 index 97986e7928..0000000000 --- a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityDropped.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.jpa.models; - -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import jakarta.persistence.Version; -import org.apache.polaris.core.entity.PolarisBaseEntity; - -/** - * EntityDropped model representing some attributes of a Polaris Entity. This is used to exchange - * entity information with ENTITIES_DROPPED table - */ -@Entity -@Table(name = "ENTITIES_DROPPED") -public class ModelEntityDropped { - // the id of the catalog associated to that entity. NULL_ID if this entity is top-level like - // a catalog - @Id private long catalogId; - - // the id of the entity which was resolved - private long id; - - // the id of the parent of this entity, use 0 for a top-level entity whose parent is the account - @Id private long parentId; - - // the type of the entity when it was resolved - @Id private int typeCode; - - // the name that this entity had when it was resolved - @Id private String name; - - // the type of the entity when it was resolved - @Id private int subTypeCode; - - // when this entity was dropped. Null if was never dropped - @Id private long dropTimestamp; - - // when should we start purging this entity - private long toPurgeTimestamp; - - // Used for Optimistic Locking to handle concurrent reads and updates - @Version private long version; - - public long getCatalogId() { - return catalogId; - } - - public long getId() { - return id; - } - - public long getParentId() { - return parentId; - } - - public int getTypeCode() { - return typeCode; - } - - public String getName() { - return name; - } - - public int getSubTypeCode() { - return subTypeCode; - } - - public long getDropTimestamp() { - return dropTimestamp; - } - - public long getToPurgeTimestamp() { - return toPurgeTimestamp; - } - - public static Builder builder() { - return new Builder(); - } - - public static final class Builder { - private final ModelEntityDropped entity; - - private Builder() { - entity = new ModelEntityDropped(); - } - - public Builder catalogId(long catalogId) { - entity.catalogId = catalogId; - return this; - } - - public Builder id(long id) { - entity.id = id; - return this; - } - - public Builder parentId(long parentId) { - entity.parentId = parentId; - return this; - } - - public Builder typeCode(int typeCode) { - entity.typeCode = typeCode; - return this; - } - - public Builder name(String name) { - entity.name = name; - return this; - } - - public Builder subTypeCode(int subTypeCode) { - entity.subTypeCode = subTypeCode; - return this; - } - - public Builder dropTimestamp(long dropTimestamp) { - entity.dropTimestamp = dropTimestamp; - return this; - } - - public Builder toPurgeTimestamp(long toPurgeTimestamp) { - entity.toPurgeTimestamp = toPurgeTimestamp; - return this; - } - - public ModelEntityDropped build() { - return entity; - } - } - - public static ModelEntityDropped fromEntity(PolarisBaseEntity entity) { - if (entity == null) return null; - - return ModelEntityDropped.builder() - .catalogId(entity.getCatalogId()) - .id(entity.getId()) - .parentId(entity.getParentId()) - .typeCode(entity.getTypeCode()) - .name(entity.getName()) - .subTypeCode(entity.getSubTypeCode()) - .dropTimestamp(entity.getDropTimestamp()) - .toPurgeTimestamp(entity.getToPurgeTimestamp()) - .build(); - } -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 6214d0f05f..3eb32d07dc 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -69,9 +69,6 @@ public class PolarisMetaStoreManagerImpl extends BaseMetaStoreManager { /** mapper, allows to serialize/deserialize properties to/from JSON */ private static final ObjectMapper MAPPER = new ObjectMapper(); - /** use synchronous drop for entities */ - private static final boolean USE_SYNCHRONOUS_DROP = true; - /** * Lookup an entity by its name * @@ -280,74 +277,53 @@ private void dropEntity( // delete it from active slice ms.deleteFromEntitiesActive(callCtx, entity); - // for now drop all entities synchronously - if (USE_SYNCHRONOUS_DROP) { - // use synchronous drop - - // delete ALL grant records to (if the entity is a grantee) and from that entity - final List grantsOnGrantee = - (entity.getType().isGrantee()) - ? ms.loadAllGrantRecordsOnGrantee(callCtx, entity.getCatalogId(), entity.getId()) - : List.of(); - final List grantsOnSecurable = - ms.loadAllGrantRecordsOnSecurable(callCtx, entity.getCatalogId(), entity.getId()); - ms.deleteAllEntityGrantRecords(callCtx, entity, grantsOnGrantee, grantsOnSecurable); - - // Now determine the set of entities on the other side of the grants we just removed. Grants - // from/to these entities has been removed, hence we need to update the grant version of - // each entity. Collect the id of each. - Set entityIdsGrantChanged = new HashSet<>(); - grantsOnGrantee.forEach( - gr -> - entityIdsGrantChanged.add( - new PolarisEntityId(gr.getSecurableCatalogId(), gr.getSecurableId()))); - grantsOnSecurable.forEach( - gr -> - entityIdsGrantChanged.add( - new PolarisEntityId(gr.getGranteeCatalogId(), gr.getGranteeId()))); - - // Bump up the grant version of these entities - List entities = - ms.lookupEntities(callCtx, new ArrayList<>(entityIdsGrantChanged)); - for (PolarisBaseEntity entityGrantChanged : entities) { - entityGrantChanged.setGrantRecordsVersion(entityGrantChanged.getGrantRecordsVersion() + 1); - ms.writeToEntities(callCtx, entityGrantChanged); - ms.writeToEntitiesChangeTracking(callCtx, entityGrantChanged); - } - - // remove the entity being dropped now - ms.deleteFromEntities(callCtx, entity); - ms.deleteFromEntitiesChangeTracking(callCtx, entity); - - // if it is a principal, we also need to drop the secrets - if (entity.getType() == PolarisEntityType.PRINCIPAL) { - // get internal properties - Map properties = - this.deserializeProperties(callCtx, entity.getInternalProperties()); - - // get client_id - String clientId = properties.get(PolarisEntityConstants.getClientIdPropertyName()); - - // delete it from the secret slice - ms.deletePrincipalSecrets(callCtx, clientId, entity.getId()); - } - } else { + // for now drop all associated grants, etc. synchronously + // delete ALL grant records to (if the entity is a grantee) and from that entity + final List grantsOnGrantee = + (entity.getType().isGrantee()) + ? ms.loadAllGrantRecordsOnGrantee(callCtx, entity.getCatalogId(), entity.getId()) + : List.of(); + final List grantsOnSecurable = + ms.loadAllGrantRecordsOnSecurable(callCtx, entity.getCatalogId(), entity.getId()); + ms.deleteAllEntityGrantRecords(callCtx, entity, grantsOnGrantee, grantsOnSecurable); + + // Now determine the set of entities on the other side of the grants we just removed. Grants + // from/to these entities has been removed, hence we need to update the grant version of + // each entity. Collect the id of each. + Set entityIdsGrantChanged = new HashSet<>(); + grantsOnGrantee.forEach( + gr -> + entityIdsGrantChanged.add( + new PolarisEntityId(gr.getSecurableCatalogId(), gr.getSecurableId()))); + grantsOnSecurable.forEach( + gr -> + entityIdsGrantChanged.add( + new PolarisEntityId(gr.getGranteeCatalogId(), gr.getGranteeId()))); + + // Bump up the grant version of these entities + List entities = + ms.lookupEntities(callCtx, new ArrayList<>(entityIdsGrantChanged)); + for (PolarisBaseEntity entityGrantChanged : entities) { + entityGrantChanged.setGrantRecordsVersion(entityGrantChanged.getGrantRecordsVersion() + 1); + ms.writeToEntities(callCtx, entityGrantChanged); + ms.writeToEntitiesChangeTracking(callCtx, entityGrantChanged); + } - // update the entity to indicate it has been dropped - final long now = System.currentTimeMillis(); - entity.setDropTimestamp(now); - entity.setLastUpdateTimestamp(now); + // remove the entity being dropped now + ms.deleteFromEntities(callCtx, entity); + ms.deleteFromEntitiesChangeTracking(callCtx, entity); - // schedule purge - entity.setToPurgeTimestamp(now + PolarisEntityConstants.getRetentionTimeInMs()); + // if it is a principal, we also need to drop the secrets + if (entity.getType() == PolarisEntityType.PRINCIPAL) { + // get internal properties + Map properties = + this.deserializeProperties(callCtx, entity.getInternalProperties()); - // increment version - entity.setEntityVersion(entity.getEntityVersion() + 1); + // get client_id + String clientId = properties.get(PolarisEntityConstants.getClientIdPropertyName()); - // write to the dropped slice and to purge slice - ms.writeToEntities(callCtx, entity); - ms.writeToEntitiesDropped(callCtx, entity); - ms.writeToEntitiesChangeTracking(callCtx, entity); + // delete it from the secret slice + ms.deletePrincipalSecrets(callCtx, clientId, entity.getId()); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index 124ec1d7a9..ee1d4f19ca 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -122,17 +122,6 @@ void runActionInReadTransaction( void writeToEntitiesActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - /** - * Write the base entity to the entities_dropped table. If there is a conflict (existing record - * with the same PK), all attributes of the new record will replace the existing one. - * - * @param callCtx call context - * @param entity entity record to write, potentially replacing an existing entity record with the - * same key - */ - void writeToEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - /** * Write the base entity to the entities change tracking table. If there is a conflict (existing * record with the same id), all attributes of the new record will replace the existing one. @@ -172,15 +161,6 @@ void writeToGrantRecords( void deleteFromEntitiesActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); - /** - * Delete the base entity to the entities_dropped table - * - * @param callCtx call context - * @param entity entity record to delete - */ - void deleteFromEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - /** * Delete the base entity from the entities change tracking table * diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index 6516b7b7fa..536e81c77e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -126,15 +126,6 @@ public void writeToEntitiesActive( this.store.getSliceEntitiesActive().write(entity); } - /** {@inheritDoc} */ - @Override - public void writeToEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // write it - this.store.getSliceEntitiesDropped().write(entity); - this.store.getSliceEntitiesDroppedToPurge().write(entity); - } - /** {@inheritDoc} */ @Override public void writeToEntitiesChangeTracking( @@ -169,15 +160,6 @@ public void deleteFromEntitiesActive( this.store.getSliceEntitiesActive().delete(this.store.buildEntitiesActiveKey(entity)); } - /** {@inheritDoc} */ - @Override - public void deleteFromEntitiesDropped( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { - // delete it - this.store.getSliceEntitiesDropped().delete(entity); - this.store.getSliceEntitiesDroppedToPurge().delete(entity); - } - /** * {@inheritDoc} * diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java index 544bcf0ffe..3d5f3d7f9c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java @@ -194,16 +194,10 @@ public boolean isWrite() { // all entities private final Slice sliceEntities; - // all entities + // all entities by-name private final Slice sliceEntitiesActive; - // all entities dropped - private final Slice sliceEntitiesDropped; - - // all entities dropped - private final Slice sliceEntitiesDroppedToPurge; - - // all entities dropped + // all entities just holding their entityVersions and grantVersions private final Slice sliceEntitiesChangeTracking; // all grant records indexed by securable @@ -231,32 +225,9 @@ public PolarisTreeMapStore(@Nonnull PolarisDiagnostics diagnostics) { entity -> String.format("%d::%d", entity.getCatalogId(), entity.getId()), PolarisBaseEntity::new); - // the entities active slice + // the entities active slice; simply acts as a name-based index into the entities slice this.sliceEntitiesActive = new Slice<>(this::buildEntitiesActiveKey, PolarisBaseEntity::new); - // the entities active slice - this.sliceEntitiesDropped = - new Slice<>( - entity -> - String.format( - "%d::%d::%s::%d::%d::%d", - entity.getCatalogId(), - entity.getParentId(), - entity.getName(), - entity.getTypeCode(), - entity.getSubTypeCode(), - entity.getDropTimestamp()), - PolarisBaseEntity::new); - - // the entities active slice - this.sliceEntitiesDroppedToPurge = - new Slice<>( - entity -> - String.format( - "%d::%d::%s", - entity.getToPurgeTimestamp(), entity.getCatalogId(), entity.getId()), - PolarisBaseEntity::new); - // change tracking this.sliceEntitiesChangeTracking = new Slice<>( @@ -370,8 +341,6 @@ private void startWriteTransaction() { this.tr = new Transaction(true); this.sliceEntities.startWriteTransaction(); this.sliceEntitiesActive.startWriteTransaction(); - this.sliceEntitiesDropped.startWriteTransaction(); - this.sliceEntitiesDroppedToPurge.startWriteTransaction(); this.sliceEntitiesChangeTracking.startWriteTransaction(); this.sliceGrantRecords.startWriteTransaction(); this.sliceGrantRecordsByGrantee.startWriteTransaction(); @@ -382,8 +351,6 @@ private void startWriteTransaction() { void rollback() { this.sliceEntities.rollback(); this.sliceEntitiesActive.rollback(); - this.sliceEntitiesDropped.rollback(); - this.sliceEntitiesDroppedToPurge.rollback(); this.sliceEntitiesChangeTracking.rollback(); this.sliceGrantRecords.rollback(); this.sliceGrantRecordsByGrantee.rollback(); @@ -510,14 +477,6 @@ public Slice getSliceEntitiesActive() { return sliceEntitiesActive; } - public Slice getSliceEntitiesDropped() { - return sliceEntitiesDropped; - } - - public Slice getSliceEntitiesDroppedToPurge() { - return sliceEntitiesDroppedToPurge; - } - public Slice getSliceEntitiesChangeTracking() { return sliceEntitiesChangeTracking; } @@ -548,8 +507,6 @@ void deleteAll() { this.ensureReadWriteTr(); this.sliceEntities.deleteAll(); this.sliceEntitiesActive.deleteAll(); - this.sliceEntitiesDropped.deleteAll(); - this.sliceEntitiesDroppedToPurge.deleteAll(); this.sliceEntitiesChangeTracking.deleteAll(); this.sliceGrantRecordsByGrantee.deleteAll(); this.sliceGrantRecords.deleteAll(); From c7c9c459fe9d22a0869030a447b7f1f623e83876 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Tue, 25 Feb 2025 06:58:12 +0000 Subject: [PATCH 06/15] Extract BasePersistence interface as parent interface of PolarisMetaStoreSession; only leave the transaction-specific methods in PolarisMetaStoreSession --- .../core/persistence/BasePersistence.java | 411 ++++++++++++++++++ .../persistence/PolarisMetaStoreSession.java | 391 +---------------- 2 files changed, 414 insertions(+), 388 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java new file mode 100644 index 0000000000..b96e1d6ddb --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; +import java.util.function.Function; +import java.util.function.Predicate; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; +import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisGrantRecord; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; + +/** + * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like + * metadata for Polaris entities and metadata about grants between these entities which is the + * foundation of our role base access control model. + * + *

Note that APIs to the actual persistence store are very basic, often point read or write to + * the underlying data store. The goal is to make it really easy to back this using databases like + * Postgres or simpler KV store. + */ +public interface BasePersistence { + /** + * @param callCtx call context + * @return new unique entity identifier + */ + long generateNewId(@Nonnull PolarisCallContext callCtx); + + /** + * Write the base entity to the entities table. If there is a conflict (existing record with the + * same id), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param entity entity record to write, potentially replacing an existing entity record with the + * same key + */ + void writeToEntities(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + + /** + * Write the specified grantRecord to the grant_records table. If there is a conflict (existing + * record with the same PK), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param grantRec entity record to write, potentially replacing an existing entity record with + * the same key + */ + void writeToGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the base entity from the entities table. + * + * @param callCtx call context + * @param entity entity record to delete + */ + void deleteFromEntities(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); + + /** + * Delete the specified grantRecord to the grant_records table. + * + * @param callCtx call context + * @param grantRec entity record to delete. + */ + void deleteFromGrantRecords( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); + + /** + * Delete the all grant records in the grant_records table for the specified entity. This method + * will delete all grant records on that securable entity and also all grants to that grantee + * entity assuming that the entity is a grantee (catalog role, principal role or principal). + * + * @param callCtx call context + * @param entity entity whose grant records to and from should be deleted + * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a + * grantee + * @param grantsOnSecurable all grants on that securable entity + */ + void deleteAllEntityGrantRecords( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisEntityCore entity, + @Nonnull List grantsOnGrantee, + @Nonnull List grantsOnSecurable); + + /** + * Delete Polaris entity and grant record metadata from all tables. This is used during metadata + * bootstrap to reset all tables to their original state + * + * @param callCtx call context + */ + void deleteAll(@Nonnull PolarisCallContext callCtx); + + /** + * Lookup an entity given its catalog id (which can be NULL_ID for top-level entities) and its + * unique id. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return NULL if the entity was not found, else the base entity. + */ + @Nullable + PolarisBaseEntity lookupEntity( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup a set of entities given their catalog id/entity id unique identifier + * + * @param callCtx call context + * @param entityIds list of entity ids + * @return list of polaris base entities, parallel to the input list of ids. An entity in the list + * will be null if the corresponding entity could not be found. + */ + @Nonnull + List lookupEntities( + @Nonnull PolarisCallContext callCtx, List entityIds); + + /** + * Lookup the current entityVersion of an entity given its catalog id (which can be NULL_ID for + * top-level entities) and its unique id. Will return 0 if the entity does not exist. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return current version for that entity or 0 if entity was not found. + */ + int lookupEntityVersion(@Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Get change tracking versions for all specified entity ids. + * + * @param callCtx call context + * @param entityIds list of entity id + * @return list parallel to the input list of entity versions. If an entity cannot be found, the + * corresponding element in the list will be null + */ + @Nonnull + List lookupEntityVersions( + @Nonnull PolarisCallContext callCtx, List entityIds); + + /** + * Lookup an entity by entityActiveKey + * + * @param callCtx call context + * @param entityActiveKey key by name + * @return null if the specified entity does not exist or has been dropped. + */ + @Nullable + PolarisEntityActiveRecord lookupEntityActive( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey); + + /** + * Lookup the specified set of entities by entityActiveKeys Return the result, a parallel list of + * active records. A record in that list will be null if its associated lookup failed + * + * @return the list of entityActiveKeys for the specified lookup operation + */ + @Nonnull + List lookupEntityActiveBatch( + @Nonnull PolarisCallContext callCtx, List entityActiveKeys); + + /** + * List all active entities of the specified type which are child entities of the specified parent + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @return the list of entities_active records for the specified list operation + */ + @Nonnull + List listActiveEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType); + + /** + * List active entities where some predicate returns true + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @param entityFilter the filter to be applied to each entity. Only entities where the predicate + * returns true are returned in the list + * @return the list of entities for which the predicate returns true + */ + @Nonnull + List listActiveEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + @Nonnull Predicate entityFilter); + + /** + * List active entities where some predicate returns true and transform the entities with a + * function + * + * @param callCtx call context + * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level + * @param parentId id of the parent, can be the special 0 value representing the root entity + * @param entityType type of entities to list + * @param limit the max number of items to return + * @param entityFilter the filter to be applied to each entity. Only entities where the predicate + * returns true are returned in the list + * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before + * returning + * @return the list of entities for which the predicate returns true + */ + @Nonnull + List listActiveEntities( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + @Nonnull PolarisEntityType entityType, + int limit, + @Nonnull Predicate entityFilter, + @Nonnull Function transformer); + + /** + * Lookup the current entityGrantRecordsVersion for the specified entity. That version is changed + * everytime a grant record is added or removed on a base securable or added to a grantee. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param entityId unique entity id + * @return current grant records version for that entity. + */ + int lookupEntityGrantRecordsVersion( + @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + + /** + * Lookup the specified grant record from the grant_records table. Return NULL if not found + * + * @param callCtx call context + * @param securableCatalogId catalog id of the securable entity, NULL_ID if the entity is + * top-level + * @param securableId id of the securable entity + * @param granteeCatalogId catalog id of the grantee entity, NULL_ID if the entity is top-level + * @param granteeId id of the grantee entity + * @param privilegeCode code for the privilege we are looking up + * @return the grant record if found, NULL if not found + */ + @Nullable + PolarisGrantRecord lookupGrantRecord( + @Nonnull PolarisCallContext callCtx, + long securableCatalogId, + long securableId, + long granteeCatalogId, + long granteeId, + int privilegeCode); + + /** + * Get all grant records on the specified securable entity. + * + * @param callCtx call context + * @param securableCatalogId catalog id of the securable entity, NULL_ID if the entity is + * top-level + * @param securableId id of the securable entity + * @return the list of grant records for the specified securable + */ + @Nonnull + List loadAllGrantRecordsOnSecurable( + @Nonnull PolarisCallContext callCtx, long securableCatalogId, long securableId); + + /** + * Get all grant records granted to the specified grantee entity. + * + * @param callCtx call context + * @param granteeCatalogId catalog id of the grantee entity, NULL_ID if the entity is top-level + * @param granteeId id of the grantee entity + * @return the list of grant records for the specified grantee + */ + @Nonnull + List loadAllGrantRecordsOnGrantee( + @Nonnull PolarisCallContext callCtx, long granteeCatalogId, long granteeId); + + /** + * Allows to retrieve to the secrets of a principal given its unique client id + * + * @param callCtx call context + * @param clientId principal client id + * @return the secrets + */ + @Nullable + PolarisPrincipalSecrets loadPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId); + + /** + * generate and store a client id and associated secrets for a newly created principal entity + * + * @param callCtx call context + * @param principalName name of the principal + * @param principalId principal id + */ + @Nonnull + PolarisPrincipalSecrets generateNewPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String principalName, long principalId); + + /** + * Rotate the secrets of a principal entity, i.e. make the specified main secrets the secondary + * and generate a new main secret + * + * @param callCtx call context + * @param clientId principal client id + * @param principalId principal id + * @param reset true if the principal secrets should be disabled and replaced with a one-time + * password + * @param oldSecretHash the principal secret's old main secret hash + */ + @Nullable + PolarisPrincipalSecrets rotatePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, + @Nonnull String clientId, + long principalId, + boolean reset, + @Nonnull String oldSecretHash); + + /** + * When dropping a principal, we also need to drop the secrets of that principal + * + * @param callCtx the call context + * @param clientId principal client id + * @param principalId the id of the principal whose secrets are dropped + */ + void deletePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId, long principalId); + + /** + * Create an in-memory storage integration + * + * @param callCtx the polaris calllctx + * @param catalogId the catalog id + * @param entityId the entity id + * @param polarisStorageConfigurationInfo the storage configuration information + * @return a storage integration object + */ + @Nullable + PolarisStorageIntegration createStorageIntegration( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long entityId, + PolarisStorageConfigurationInfo polarisStorageConfigurationInfo); + + /** + * Persist a storage integration in the metastore + * + * @param callContext the polaris call context + * @param entity the entity of the object + * @param storageIntegration the storage integration to persist + */ + void persistStorageIntegrationIfNeeded( + @Nonnull PolarisCallContext callContext, + @Nonnull PolarisBaseEntity entity, + @Nullable PolarisStorageIntegration storageIntegration); + + /** + * Load the polaris storage integration for a polaris entity (Catalog,Namespace,Table,View) + * + * @param callContext the polaris call context + * @param entity the polaris entity + * @return a polaris storage integration + */ + @Nullable + + PolarisStorageIntegration loadPolarisStorageIntegration( + @Nonnull PolarisCallContext callContext, @Nonnull PolarisBaseEntity entity); + + /** + * Check if the specified parent entity has children. + * + * @param callContext the polaris call context + * @param optionalEntityType if not null, only check for the specified type, else check for all + * types of children entities + * @param catalogId id of the catalog + * @param parentId id of the parent, either a namespace or a catalog + * @return true if the parent entity has children + */ + boolean hasChildren( + @Nonnull PolarisCallContext callContext, + @Nullable PolarisEntityType optionalEntityType, + long catalogId, + long parentId); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index ee1d4f19ca..208122f032 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -19,34 +19,16 @@ package org.apache.polaris.core.persistence; import jakarta.annotation.Nonnull; -import jakarta.annotation.Nullable; -import java.util.List; -import java.util.function.Function; -import java.util.function.Predicate; import java.util.function.Supplier; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; -import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; -import org.apache.polaris.core.entity.PolarisEntityId; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; -import org.apache.polaris.core.storage.PolarisStorageIntegration; /** - * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like - * metadata for Polaris entities and metadata about grants between these entities which is the - * foundation of our role base access control model. - * - *

Note that APIs to the actual persistence store are very basic, often point read or write to - * the underlying data store. The goal is to make it really easy to back this using databases like - * Postgres or simpler KV store. + * Extends BasePersistence to express a more "transaction-oriented" control flow for backing stores + * which can support a runInTransaction semantic. */ -public interface PolarisMetaStoreSession { +public interface PolarisMetaStoreSession extends BasePersistence { /** * Run the specified transaction code (a Supplier lambda type) in a database read/write @@ -95,22 +77,6 @@ T runInReadTransaction( void runActionInReadTransaction( @Nonnull PolarisCallContext callCtx, @Nonnull Runnable transactionCode); - /** - * @param callCtx call context - * @return new unique entity identifier - */ - long generateNewId(@Nonnull PolarisCallContext callCtx); - - /** - * Write the base entity to the entities table. If there is a conflict (existing record with the - * same id), all attributes of the new record will replace the existing one. - * - * @param callCtx call context - * @param entity entity record to write, potentially replacing an existing entity record with the - * same key - */ - void writeToEntities(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - /** * Write the base entity to the entities_active table. If there is a conflict (existing record * with the same PK), all attributes of the new record will replace the existing one. @@ -133,25 +99,6 @@ void writeToEntitiesActive( void writeToEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); - /** - * Write the specified grantRecord to the grant_records table. If there is a conflict (existing - * record with the same PK), all attributes of the new record will replace the existing one. - * - * @param callCtx call context - * @param grantRec entity record to write, potentially replacing an existing entity record with - * the same key - */ - void writeToGrantRecords( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); - - /** - * Delete the base entity from the entities table. - * - * @param callCtx call context - * @param entity entity record to delete - */ - void deleteFromEntities(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); - /** * Delete the base entity from the entities_active table. * @@ -170,338 +117,6 @@ void deleteFromEntitiesActive( void deleteFromEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); - /** - * Delete the specified grantRecord to the grant_records table. - * - * @param callCtx call context - * @param grantRec entity record to delete. - */ - void deleteFromGrantRecords( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); - - /** - * Delete the all grant records in the grant_records table for the specified entity. This method - * will delete all grant records on that securable entity and also all grants to that grantee - * entity assuming that the entity is a grantee (catalog role, principal role or principal). - * - * @param callCtx call context - * @param entity entity whose grant records to and from should be deleted - * @param grantsOnGrantee all grants to that grantee entity. Empty list if that entity is not a - * grantee - * @param grantsOnSecurable all grants on that securable entity - */ - void deleteAllEntityGrantRecords( - @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisEntityCore entity, - @Nonnull List grantsOnGrantee, - @Nonnull List grantsOnSecurable); - - /** - * Delete Polaris entity and grant record metadata from all tables. This is used during metadata - * bootstrap to reset all tables to their original state - * - * @param callCtx call context - */ - void deleteAll(@Nonnull PolarisCallContext callCtx); - - /** - * Lookup an entity given its catalog id (which can be NULL_ID for top-level entities) and its - * unique id. - * - * @param callCtx call context - * @param catalogId catalog id or NULL_ID - * @param entityId unique entity id - * @return NULL if the entity was not found, else the base entity. - */ - @Nullable - PolarisBaseEntity lookupEntity( - @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); - - /** - * Lookup a set of entities given their catalog id/entity id unique identifier - * - * @param callCtx call context - * @param entityIds list of entity ids - * @return list of polaris base entities, parallel to the input list of ids. An entity in the list - * will be null if the corresponding entity could not be found. - */ - @Nonnull - List lookupEntities( - @Nonnull PolarisCallContext callCtx, List entityIds); - - /** - * Lookup in the entities_change_tracking table the current version of an entity given its catalog - * id (which can be NULL_ID for top-level entities) and its unique id. Will return 0 if the entity - * does not exist. - * - * @param callCtx call context - * @param catalogId catalog id or NULL_ID - * @param entityId unique entity id - * @return current version for that entity or 0 if entity was not found. - */ - int lookupEntityVersion(@Nonnull PolarisCallContext callCtx, long catalogId, long entityId); - - /** - * Get change tracking versions for all specified entity ids. - * - * @param callCtx call context - * @param entityIds list of entity id - * @return list parallel to the input list of entity versions. If an entity cannot be found, the - * corresponding element in the list will be null - */ - @Nonnull - List lookupEntityVersions( - @Nonnull PolarisCallContext callCtx, List entityIds); - - /** - * Lookup in the entities_active table to determine if the specified entity exists. Return the - * result of that lookup - * - * @param callCtx call context - * @param entityActiveKey key in the ENTITIES_ACTIVE table - * @return null if the specified entity does not exist or has been dropped. - */ - @Nullable - PolarisEntityActiveRecord lookupEntityActive( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey); - - /** - * Lookup in the entities_active table to determine if the specified set of entities exist. Return - * the result, a parallel list of active records. A record in that list will be null if its - * associated lookup failed - * - * @return the list of entities_active records for the specified lookup operation - */ - @Nonnull - List lookupEntityActiveBatch( - @Nonnull PolarisCallContext callCtx, List entityActiveKeys); - - /** - * List all active entities of the specified type which are child entities of the specified parent - * - * @param callCtx call context - * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level - * @param parentId id of the parent, can be the special 0 value representing the root entity - * @param entityType type of entities to list - * @return the list of entities_active records for the specified list operation - */ - @Nonnull - List listActiveEntities( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType); - - /** - * List active entities where some predicate returns true - * - * @param callCtx call context - * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level - * @param parentId id of the parent, can be the special 0 value representing the root entity - * @param entityType type of entities to list - * @param entityFilter the filter to be applied to each entity. Only entities where the predicate - * returns true are returned in the list - * @return the list of entities for which the predicate returns true - */ - @Nonnull - List listActiveEntities( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull Predicate entityFilter); - - /** - * List active entities where some predicate returns true and transform the entities with a - * function - * - * @param callCtx call context - * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level - * @param parentId id of the parent, can be the special 0 value representing the root entity - * @param entityType type of entities to list - * @param limit the max number of items to return - * @param entityFilter the filter to be applied to each entity. Only entities where the predicate - * returns true are returned in the list - * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before - * returning - * @return the list of entities for which the predicate returns true - */ - @Nonnull - List listActiveEntities( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - int limit, - @Nonnull Predicate entityFilter, - @Nonnull Function transformer); - - /** - * Lookup in the entities_change_tracking table the current version of the grant records for this - * entity. That version is changed everytime a grant record is added or removed on a base - * securable or added to a grantee. - * - * @param callCtx call context - * @param catalogId catalog id or NULL_ID - * @param entityId unique entity id - * @return current grant records version for that entity. - */ - int lookupEntityGrantRecordsVersion( - @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); - - /** - * Lookup the specified grant record from the grant_records table. Return NULL if not found - * - * @param callCtx call context - * @param securableCatalogId catalog id of the securable entity, NULL_ID if the entity is - * top-level - * @param securableId id of the securable entity - * @param granteeCatalogId catalog id of the grantee entity, NULL_ID if the entity is top-level - * @param granteeId id of the grantee entity - * @param privilegeCode code for the privilege we are looking up - * @return the grant record if found, NULL if not found - */ - @Nullable - PolarisGrantRecord lookupGrantRecord( - @Nonnull PolarisCallContext callCtx, - long securableCatalogId, - long securableId, - long granteeCatalogId, - long granteeId, - int privilegeCode); - - /** - * Get all grant records on the specified securable entity. - * - * @param callCtx call context - * @param securableCatalogId catalog id of the securable entity, NULL_ID if the entity is - * top-level - * @param securableId id of the securable entity - * @return the list of grant records for the specified securable - */ - @Nonnull - List loadAllGrantRecordsOnSecurable( - @Nonnull PolarisCallContext callCtx, long securableCatalogId, long securableId); - - /** - * Get all grant records granted to the specified grantee entity. - * - * @param callCtx call context - * @param granteeCatalogId catalog id of the grantee entity, NULL_ID if the entity is top-level - * @param granteeId id of the grantee entity - * @return the list of grant records for the specified grantee - */ - @Nonnull - List loadAllGrantRecordsOnGrantee( - @Nonnull PolarisCallContext callCtx, long granteeCatalogId, long granteeId); - - /** - * Allows to retrieve to the secrets of a principal given its unique client id - * - * @param callCtx call context - * @param clientId principal client id - * @return the secrets - */ - @Nullable - PolarisPrincipalSecrets loadPrincipalSecrets( - @Nonnull PolarisCallContext callCtx, @Nonnull String clientId); - - /** - * generate and store a client id and associated secrets for a newly created principal entity - * - * @param callCtx call context - * @param principalName name of the principal - * @param principalId principal id - */ - @Nonnull - PolarisPrincipalSecrets generateNewPrincipalSecrets( - @Nonnull PolarisCallContext callCtx, @Nonnull String principalName, long principalId); - - /** - * Rotate the secrets of a principal entity, i.e. make the specified main secrets the secondary - * and generate a new main secret - * - * @param callCtx call context - * @param clientId principal client id - * @param principalId principal id - * @param reset true if the principal secrets should be disabled and replaced with a one-time - * password - * @param oldSecretHash the principal secret's old main secret hash - */ - @Nullable - PolarisPrincipalSecrets rotatePrincipalSecrets( - @Nonnull PolarisCallContext callCtx, - @Nonnull String clientId, - long principalId, - boolean reset, - @Nonnull String oldSecretHash); - - /** - * When dropping a principal, we also need to drop the secrets of that principal - * - * @param callCtx the call context - * @param clientId principal client id - * @param principalId the id of the principal whose secrets are dropped - */ - void deletePrincipalSecrets( - @Nonnull PolarisCallContext callCtx, @Nonnull String clientId, long principalId); - - /** - * Create an in-memory storage integration - * - * @param callCtx the polaris calllctx - * @param catalogId the catalog id - * @param entityId the entity id - * @param polarisStorageConfigurationInfo the storage configuration information - * @return a storage integration object - */ - @Nullable - PolarisStorageIntegration createStorageIntegration( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long entityId, - PolarisStorageConfigurationInfo polarisStorageConfigurationInfo); - - /** - * Persist a storage integration in the metastore - * - * @param callContext the polaris call context - * @param entity the entity of the object - * @param storageIntegration the storage integration to persist - */ - void persistStorageIntegrationIfNeeded( - @Nonnull PolarisCallContext callContext, - @Nonnull PolarisBaseEntity entity, - @Nullable PolarisStorageIntegration storageIntegration); - - /** - * Load the polaris storage integration for a polaris entity (Catalog,Namespace,Table,View) - * - * @param callContext the polaris call context - * @param entity the polaris entity - * @return a polaris storage integration - */ - @Nullable - - PolarisStorageIntegration loadPolarisStorageIntegration( - @Nonnull PolarisCallContext callContext, @Nonnull PolarisBaseEntity entity); - - /** - * Check if the specified parent entity has children. - * - * @param callContext the polaris call context - * @param optionalEntityType if not null, only check for the specified type, else check for all - * types of children entities - * @param catalogId id of the catalog - * @param parentId id of the parent, either a namespace or a catalog - * @return true if the parent entity has children - */ - boolean hasChildren( - @Nonnull PolarisCallContext callContext, - @Nullable PolarisEntityType optionalEntityType, - long catalogId, - long parentId); - /** Rollback the current transaction */ void rollback(); } From 87a8010fdb32ceb872edd7c128a45a70230214fb Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Wed, 26 Feb 2025 05:17:50 +0000 Subject: [PATCH 07/15] Push all evidence of the two-phase lookupEntityByName into only the transactional-style PolarisMetaStoreSession, so that BasePersistence properly exposes a lookupEntityByName method where impls that use secondary indexes can easily just lookup an entity by name instead of doing two lookups. --- .../core/persistence/BasePersistence.java | 40 +++++------ .../PolarisMetaStoreManagerImpl.java | 68 +++++-------------- .../persistence/PolarisMetaStoreSession.java | 56 +++++++++++++++ 3 files changed, 92 insertions(+), 72 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java index b96e1d6ddb..a4283bf5f8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -26,7 +26,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; -import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; @@ -128,6 +127,24 @@ void deleteAllEntityGrantRecords( PolarisBaseEntity lookupEntity( @Nonnull PolarisCallContext callCtx, long catalogId, long entityId); + /** + * Lookup an entity given its catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param parentId id of the parent, either a namespace or a catalog + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + @Nullable + PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name); + /** * Lookup a set of entities given their catalog id/entity id unique identifier * @@ -163,27 +180,6 @@ List lookupEntities( List lookupEntityVersions( @Nonnull PolarisCallContext callCtx, List entityIds); - /** - * Lookup an entity by entityActiveKey - * - * @param callCtx call context - * @param entityActiveKey key by name - * @return null if the specified entity does not exist or has been dropped. - */ - @Nullable - PolarisEntityActiveRecord lookupEntityActive( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey); - - /** - * Lookup the specified set of entities by entityActiveKeys Return the result, a parallel list of - * active records. A record in that list will be null if its associated lookup failed - * - * @return the list of entityActiveKeys for the specified lookup operation - */ - @Nonnull - List lookupEntityActiveBatch( - @Nonnull PolarisCallContext callCtx, List entityActiveKeys); - /** * List all active entities of the specified type which are child entities of the specified parent * diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 3eb32d07dc..7a0168543c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -69,38 +69,6 @@ public class PolarisMetaStoreManagerImpl extends BaseMetaStoreManager { /** mapper, allows to serialize/deserialize properties to/from JSON */ private static final ObjectMapper MAPPER = new ObjectMapper(); - /** - * Lookup an entity by its name - * - * @param callCtx call context - * @param ms meta store - * @param entityActiveKey lookup key - * @return the entity if it exists, null otherwise - */ - private @Nullable PolarisBaseEntity lookupEntityByName( - @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, - @Nonnull PolarisEntitiesActiveKey entityActiveKey) { - // ensure that the entity exists - PolarisEntityActiveRecord entityActiveRecord = ms.lookupEntityActive(callCtx, entityActiveKey); - - // if not found, return null - if (entityActiveRecord == null) { - return null; - } - - // lookup the entity, should be there - PolarisBaseEntity entity = - ms.lookupEntity(callCtx, entityActiveRecord.getCatalogId(), entityActiveRecord.getId()); - callCtx - .getDiagServices() - .checkNotNull( - entity, "unexpected_not_found_entity", "entityActiveRecord={}", entityActiveRecord); - - // return it now - return entity; - } - /** * Write this entity to the meta store. * @@ -512,13 +480,13 @@ private void revokeGrantRecord( catalog); // lookup catalog admin role, should exist - PolarisEntitiesActiveKey adminRoleKey = - new PolarisEntitiesActiveKey( + PolarisBaseEntity catalogAdminRole = + ms.lookupEntityByName( + callCtx, refreshCatalog.getId(), refreshCatalog.getId(), PolarisEntityType.CATALOG_ROLE.getCode(), PolarisEntityConstants.getNameOfCatalogAdminRole()); - PolarisBaseEntity catalogAdminRole = this.lookupEntityByName(callCtx, ms, adminRoleKey); // if found, ensure not null callCtx @@ -573,14 +541,13 @@ private void revokeGrantRecord( // immediately assign its catalog_admin role if (principalRoles.isEmpty()) { // lookup service admin role, should exist - PolarisEntitiesActiveKey serviceAdminRoleKey = - new PolarisEntitiesActiveKey( + PolarisBaseEntity serviceAdminRole = + ms.lookupEntityByName( + callCtx, PolarisEntityConstants.getNullId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.PRINCIPAL_ROLE.getCode(), PolarisEntityConstants.getNameOfPrincipalServiceAdminRole()); - PolarisBaseEntity serviceAdminRole = - this.lookupEntityByName(callCtx, ms, serviceAdminRoleKey); callCtx.getDiagServices().checkNotNull(serviceAdminRole, "missing_service_admin_role"); this.persistNewGrantRecord( callCtx, ms, adminRole, serviceAdminRole, PolarisPrivilege.CATALOG_ROLE_USAGE); @@ -719,10 +686,13 @@ private void bootstrapPolarisService( } // now looking the entity by name - PolarisEntitiesActiveKey entityActiveKey = - new PolarisEntitiesActiveKey( - resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType.getCode(), name); - PolarisBaseEntity entity = this.lookupEntityByName(callCtx, ms, entityActiveKey); + PolarisBaseEntity entity = + ms.lookupEntityByName( + callCtx, + resolver.getCatalogIdOrNull(), + resolver.getParentId(), + entityType.getCode(), + name); // if found, check if subType really matches if (entity != null @@ -2218,9 +2188,8 @@ public Map getInternalPropertyMap( @Nonnull String entityName) { // load that entity - PolarisEntitiesActiveKey entityActiveKey = - new PolarisEntitiesActiveKey(entityCatalogId, parentId, entityType.getCode(), entityName); - PolarisBaseEntity entity = this.lookupEntityByName(callCtx, ms, entityActiveKey); + PolarisBaseEntity entity = + ms.lookupEntityByName(callCtx, entityCatalogId, parentId, entityType.getCode(), entityName); // null if entity not found if (entity == null) { @@ -2279,14 +2248,13 @@ public Map getInternalPropertyMap( EntityResult backfillResult = this.createEntityIfNotExists(callCtx, ms, null, rootContainer); if (backfillResult.isSuccess()) { - PolarisEntitiesActiveKey serviceAdminRoleKey = - new PolarisEntitiesActiveKey( + PolarisBaseEntity serviceAdminRole = + ms.lookupEntityByName( + callCtx, 0L, 0L, PolarisEntityType.PRINCIPAL_ROLE.getCode(), PolarisEntityConstants.getNameOfPrincipalServiceAdminRole()); - PolarisBaseEntity serviceAdminRole = - this.lookupEntityByName(callCtx, ms, serviceAdminRoleKey); if (serviceAdminRole != null) { this.persistNewGrantRecord( callCtx, diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index 208122f032..3a871ffb66 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -19,9 +19,13 @@ package org.apache.polaris.core.persistence; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.List; import java.util.function.Supplier; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; +import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; /** @@ -77,6 +81,58 @@ T runInReadTransaction( void runActionInReadTransaction( @Nonnull PolarisCallContext callCtx, @Nonnull Runnable transactionCode); + /** {@inheritDoc} */ + @Override + default PolarisBaseEntity lookupEntityByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name) { + PolarisEntitiesActiveKey entityActiveKey = + new PolarisEntitiesActiveKey(catalogId, parentId, typeCode, name); + + // ensure that the entity exists + PolarisEntityActiveRecord entityActiveRecord = lookupEntityActive(callCtx, entityActiveKey); + + // if not found, return null + if (entityActiveRecord == null) { + return null; + } + + // lookup the entity, should be there + PolarisBaseEntity entity = + lookupEntity(callCtx, entityActiveRecord.getCatalogId(), entityActiveRecord.getId()); + callCtx + .getDiagServices() + .checkNotNull( + entity, "unexpected_not_found_entity", "entityActiveRecord={}", entityActiveRecord); + + // return it now + return entity; + } + + /** + * Lookup an entity by entityActiveKey + * + * @param callCtx call context + * @param entityActiveKey key by name + * @return null if the specified entity does not exist or has been dropped. + */ + @Nullable + PolarisEntityActiveRecord lookupEntityActive( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey); + + /** + * Lookup the specified set of entities by entityActiveKeys Return the result, a parallel list of + * active records. A record in that list will be null if its associated lookup failed + * + * @return the list of entityActiveKeys for the specified lookup operation + */ + @Nonnull + List lookupEntityActiveBatch( + @Nonnull PolarisCallContext callCtx, List entityActiveKeys); + /** * Write the base entity to the entities_active table. If there is a conflict (existing record * with the same PK), all attributes of the new record will replace the existing one. From 13ba1616c0e4a6f434e7d8befc6bb6e3c89c50c9 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Wed, 26 Feb 2025 06:46:50 +0000 Subject: [PATCH 08/15] Turn PolarisMetaStoreSession into an abstract class and make lookupEntityActive protected-visibility; remove all callsites where PolarisMetaStoreManagerImpl calls it. Technically, while in the same package this doesn't prevent it from leaking, but we could reposition PolarisMetaStoreSession into a separate transaction-specific package to help protect it from leaking the lower-level abstractions. --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 2 +- .../core/persistence/BasePersistence.java | 23 ++++++++++ .../PolarisMetaStoreManagerImpl.java | 40 ++++++++--------- .../persistence/PolarisMetaStoreSession.java | 43 +++++++++++++------ .../PolarisTreeMapMetaStoreSessionImpl.java | 2 +- 5 files changed, 71 insertions(+), 39 deletions(-) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 621993e7fb..7de38105a6 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -68,7 +68,7 @@ * EclipseLink implementation of a Polaris metadata store supporting persisting and retrieving all * Polaris metadata from/to the configured database systems. */ -public class PolarisEclipseLinkMetaStoreSessionImpl implements PolarisMetaStoreSession { +public class PolarisEclipseLinkMetaStoreSessionImpl extends PolarisMetaStoreSession { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisEclipseLinkMetaStoreSessionImpl.class); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java index a4283bf5f8..9ac8268b53 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -145,6 +145,29 @@ PolarisBaseEntity lookupEntityByName( int typeCode, @Nonnull String name); + /** + * Looks up just the entity's subType and id given it catalogId, parentId, typeCode, and name. + * + * @param callCtx call context + * @param catalogId catalog id or NULL_ID + * @param parentId id of the parent, either a namespace or a catalog + * @param typeCode the PolarisEntityType code of the entity to lookup + * @param name the name of the entity + * @return null if the specified entity does not exist + */ + default PolarisEntityActiveRecord lookupEntityIdAndSubTypeByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name) { + PolarisBaseEntity baseEntity = lookupEntityByName(callCtx, catalogId, parentId, typeCode, name); + if (baseEntity == null) { + return null; + } + return new PolarisEntityActiveRecord(baseEntity); + } + /** * Lookup a set of entities given their catalog id/entity id unique identifier * diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 7a0168543c..9fa9251403 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -39,7 +39,6 @@ import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; -import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityConstants; @@ -499,16 +498,14 @@ private void revokeGrantRecord( } // check that a catalog with the same name does not exist already - PolarisEntitiesActiveKey catalogNameKey = - new PolarisEntitiesActiveKey( + // if it exists, this is an error, the client should retry + if (ms.lookupEntityIdAndSubTypeByName( + callCtx, PolarisEntityConstants.getNullId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.CATALOG.getCode(), - catalog.getName()); - PolarisEntityActiveRecord otherCatalogRecord = ms.lookupEntityActive(callCtx, catalogNameKey); - - // if it exists, this is an error, the client should retry - if (otherCatalogRecord != null) { + catalog.getName()) + != null) { return new CreateCatalogResult(BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, null); } @@ -886,17 +883,14 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } // check that a principal with the same name does not exist already - PolarisEntitiesActiveKey principalNameKey = - new PolarisEntitiesActiveKey( + // if it exists, this is an error, the client should retry + if (ms.lookupEntityIdAndSubTypeByName( + callCtx, PolarisEntityConstants.getNullId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.PRINCIPAL.getCode(), - principal.getName()); - PolarisEntityActiveRecord otherPrincipalRecord = - ms.lookupEntityActive(callCtx, principalNameKey); - - // if it exists, this is an error, the client should retry - if (otherPrincipalRecord != null) { + principal.getName()) + != null) { return new CreatePrincipalResult(BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, null); } @@ -1087,13 +1081,13 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } // check if an entity does not already exist with the same name. If true, this is an error - PolarisEntitiesActiveKey entityActiveKey = - new PolarisEntitiesActiveKey( + PolarisEntityActiveRecord entityActiveRecord = + ms.lookupEntityIdAndSubTypeByName( + callCtx, entity.getCatalogId(), entity.getParentId(), entity.getType().getCode(), entity.getName()); - PolarisEntityActiveRecord entityActiveRecord = ms.lookupEntityActive(callCtx, entityActiveKey); if (entityActiveRecord != null) { return new EntityResult( BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, entityActiveRecord.getSubTypeCode()); @@ -1317,14 +1311,14 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } // ensure that nothing exists where we create that entity - PolarisEntitiesActiveKey entityActiveKey = - new PolarisEntitiesActiveKey( + // if this entity already exists, this is an error + PolarisEntityActiveRecord entityActiveRecord = + ms.lookupEntityIdAndSubTypeByName( + callCtx, resolver.getCatalogIdOrNull(), resolver.getParentId(), refreshEntityToRename.getTypeCode(), renamedEntity.getName()); - // if this entity already exists, this is an error - PolarisEntityActiveRecord entityActiveRecord = ms.lookupEntityActive(callCtx, entityActiveKey); if (entityActiveRecord != null) { return new EntityResult( BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, entityActiveRecord.getSubTypeCode()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index 3a871ffb66..883ac0f282 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -30,9 +30,10 @@ /** * Extends BasePersistence to express a more "transaction-oriented" control flow for backing stores - * which can support a runInTransaction semantic. + * which can support a runInTransaction semantic, while providing default implementations of some of + * the BasePersistence methods in terms of lower-level methods that subclasses must implement. */ -public interface PolarisMetaStoreSession extends BasePersistence { +public abstract class PolarisMetaStoreSession implements BasePersistence { /** * Run the specified transaction code (a Supplier lambda type) in a database read/write @@ -44,7 +45,8 @@ public interface PolarisMetaStoreSession extends BasePersistence { * @param callCtx call context * @param transactionCode code of the transaction being executed, a supplier lambda */ - T runInTransaction(@Nonnull PolarisCallContext callCtx, @Nonnull Supplier transactionCode); + public abstract T runInTransaction( + @Nonnull PolarisCallContext callCtx, @Nonnull Supplier transactionCode); /** * Run the specified transaction code (a runnable lambda type) in a database read/write @@ -55,7 +57,7 @@ public interface PolarisMetaStoreSession extends BasePersistence { * @param callCtx call context * @param transactionCode code of the transaction being executed, a runnable lambda */ - void runActionInTransaction( + public abstract void runActionInTransaction( @Nonnull PolarisCallContext callCtx, @Nonnull Runnable transactionCode); /** @@ -67,7 +69,7 @@ void runActionInTransaction( * @param callCtx call context * @param transactionCode code of the transaction being executed, a supplier lambda */ - T runInReadTransaction( + public abstract T runInReadTransaction( @Nonnull PolarisCallContext callCtx, @Nonnull Supplier transactionCode); /** @@ -78,12 +80,12 @@ T runInReadTransaction( * @param callCtx call context * @param transactionCode code of the transaction being executed, a runnable lambda */ - void runActionInReadTransaction( + public abstract void runActionInReadTransaction( @Nonnull PolarisCallContext callCtx, @Nonnull Runnable transactionCode); /** {@inheritDoc} */ @Override - default PolarisBaseEntity lookupEntityByName( + public PolarisBaseEntity lookupEntityByName( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @@ -112,6 +114,19 @@ default PolarisBaseEntity lookupEntityByName( return entity; } + /** {@inheritDoc} */ + @Override + public PolarisEntityActiveRecord lookupEntityIdAndSubTypeByName( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long parentId, + int typeCode, + @Nonnull String name) { + PolarisEntitiesActiveKey entityActiveKey = + new PolarisEntitiesActiveKey(catalogId, parentId, typeCode, name); + return lookupEntityActive(callCtx, entityActiveKey); + } + /** * Lookup an entity by entityActiveKey * @@ -120,7 +135,7 @@ default PolarisBaseEntity lookupEntityByName( * @return null if the specified entity does not exist or has been dropped. */ @Nullable - PolarisEntityActiveRecord lookupEntityActive( + protected abstract PolarisEntityActiveRecord lookupEntityActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey); /** @@ -130,7 +145,7 @@ PolarisEntityActiveRecord lookupEntityActive( * @return the list of entityActiveKeys for the specified lookup operation */ @Nonnull - List lookupEntityActiveBatch( + public abstract List lookupEntityActiveBatch( @Nonnull PolarisCallContext callCtx, List entityActiveKeys); /** @@ -141,7 +156,7 @@ List lookupEntityActiveBatch( * @param entity entity record to write, potentially replacing an existing entity record with the * same key */ - void writeToEntitiesActive( + public abstract void writeToEntitiesActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); /** @@ -152,7 +167,7 @@ void writeToEntitiesActive( * @param entity entity record to write, potentially replacing an existing entity record with the * same key */ - void writeToEntitiesChangeTracking( + public abstract void writeToEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); /** @@ -161,7 +176,7 @@ void writeToEntitiesChangeTracking( * @param callCtx call context * @param entity entity record to delete */ - void deleteFromEntitiesActive( + public abstract void deleteFromEntitiesActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); /** @@ -170,9 +185,9 @@ void deleteFromEntitiesActive( * @param callCtx call context * @param entity entity record to delete */ - void deleteFromEntitiesChangeTracking( + public abstract void deleteFromEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); /** Rollback the current transaction */ - void rollback(); + public abstract void rollback(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index 536e81c77e..8e29bc248f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -40,7 +40,7 @@ import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -public class PolarisTreeMapMetaStoreSessionImpl implements PolarisMetaStoreSession { +public class PolarisTreeMapMetaStoreSessionImpl extends PolarisMetaStoreSession { // the TreeMap store to use private final PolarisTreeMapStore store; From 5743dd260d0c8b659e2a2a48032138b5f642a522 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Wed, 26 Feb 2025 07:39:36 +0000 Subject: [PATCH 09/15] Pushdown all calls to writeToEntities into PolarisMetaStoreSession, and add writeEntity method to BasePersistence, with a default impl in PolarisMetaStoreSession containing what was previously in PolarisMetaStoreManagerImpl. This now protects all writes in PolarisMetaStoreManagerImpl from dealing with the three-table implementation detail. Technically slightly changes the ordering of updates within a transaction for renameEntity, but is arguably a more correct ordering, and the ordering doesn't interleave reads anyways. --- .../core/persistence/BasePersistence.java | 12 ++-- .../PolarisMetaStoreManagerImpl.java | 59 ++++++------------- .../persistence/PolarisMetaStoreSession.java | 33 ++++++++++- 3 files changed, 57 insertions(+), 47 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java index 9ac8268b53..0e9e2fab89 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -52,14 +52,16 @@ public interface BasePersistence { long generateNewId(@Nonnull PolarisCallContext callCtx); /** - * Write the base entity to the entities table. If there is a conflict (existing record with the - * same id), all attributes of the new record will replace the existing one. + * Write this entity to the meta store. * * @param callCtx call context - * @param entity entity record to write, potentially replacing an existing entity record with the - * same key + * @param entity entity to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable */ - void writeToEntities(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged); /** * Write the specified grantRecord to the grant_records table. If there is a conflict (existing diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 9fa9251403..cafb828ca3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -68,27 +68,6 @@ public class PolarisMetaStoreManagerImpl extends BaseMetaStoreManager { /** mapper, allows to serialize/deserialize properties to/from JSON */ private static final ObjectMapper MAPPER = new ObjectMapper(); - /** - * Write this entity to the meta store. - * - * @param callCtx call context - * @param ms meta store in read/write mode - * @param entity entity to persist - * @param writeToActive if true, write it to active - */ - private void writeEntity( - @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, - @Nonnull PolarisBaseEntity entity, - boolean writeToActive) { - ms.writeToEntities(callCtx, entity); - ms.writeToEntitiesChangeTracking(callCtx, entity); - - if (writeToActive) { - ms.writeToEntitiesActive(callCtx, entity); - } - } - /** * Persist the specified new entity. Persist will write this entity in the ENTITIES, in the * ENTITIES_ACTIVE and finally in the ENTITIES_CHANGE_TRACKING tables @@ -150,7 +129,7 @@ private void persistNewEntity( entity.setToPurgeTimestamp(0); // write it - this.writeEntity(callCtx, ms, entity, true); + ms.writeEntity(callCtx, entity, true); } /** @@ -161,12 +140,14 @@ private void persistNewEntity( * @param callCtx call context * @param ms meta store * @param entity the entity which has been changed + * @param nameOrParentChanged indicates if parent or name changed * @return the entity with its version and lastUpdateTimestamp updated */ private @Nonnull PolarisBaseEntity persistEntityAfterChange( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisMetaStoreSession ms, - @Nonnull PolarisBaseEntity entity) { + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged) { // validate the entity type and subtype callCtx.getDiagServices().checkNotNull(entity, "unexpected_null_entity"); @@ -209,7 +190,7 @@ private void persistNewEntity( entity.setEntityVersion(entity.getEntityVersion() + 1); // persist it to the various slices - this.writeEntity(callCtx, ms, entity, false); + ms.writeEntity(callCtx, entity, nameOrParentChanged); // return it return entity; @@ -272,8 +253,7 @@ private void dropEntity( ms.lookupEntities(callCtx, new ArrayList<>(entityIdsGrantChanged)); for (PolarisBaseEntity entityGrantChanged : entities) { entityGrantChanged.setGrantRecordsVersion(entityGrantChanged.getGrantRecordsVersion() + 1); - ms.writeToEntities(callCtx, entityGrantChanged); - ms.writeToEntitiesChangeTracking(callCtx, entityGrantChanged); + ms.writeEntity(callCtx, entityGrantChanged, false); } // remove the entity being dropped now @@ -344,7 +324,7 @@ private void dropEntity( // grants have changed, we need to bump-up the grants version granteeEntity.setGrantRecordsVersion(granteeEntity.getGrantRecordsVersion() + 1); - this.writeEntity(callCtx, ms, granteeEntity, false); + ms.writeEntity(callCtx, granteeEntity, false); // we also need to invalidate the grants on that securable so that we can reload them. // load the securable and increment its grants version @@ -356,7 +336,7 @@ private void dropEntity( // grants have changed, we need to bump-up the grants version securableEntity.setGrantRecordsVersion(securableEntity.getGrantRecordsVersion() + 1); - this.writeEntity(callCtx, ms, securableEntity, false); + ms.writeEntity(callCtx, securableEntity, false); // done, return the new grant record return grantRecord; @@ -419,7 +399,7 @@ private void revokeGrantRecord( // grants have changed, we need to bump-up the grants version refreshGrantee.setGrantRecordsVersion(refreshGrantee.getGrantRecordsVersion() + 1); - this.writeEntity(callCtx, ms, refreshGrantee, false); + ms.writeEntity(callCtx, refreshGrantee, false); // we also need to invalidate the grants on that securable so that we can reload them. // load the securable and increment its grants version @@ -436,7 +416,7 @@ private void revokeGrantRecord( // grants have changed, we need to bump-up the grants version refreshSecurable.setGrantRecordsVersion(refreshSecurable.getGrantRecordsVersion() + 1); - this.writeEntity(callCtx, ms, refreshSecurable, false); + ms.writeEntity(callCtx, refreshSecurable, false); } /** @@ -983,14 +963,14 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str principal.setInternalProperties( PolarisObjectMapperUtil.serializeProperties(callCtx, internalProps)); principal.setEntityVersion(principal.getEntityVersion() + 1); - writeEntity(callCtx, ms, principal, true); + ms.writeEntity(callCtx, principal, true); } else if (internalProps.containsKey( PolarisEntityConstants.PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE)) { internalProps.remove(PolarisEntityConstants.PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE); principal.setInternalProperties( PolarisObjectMapperUtil.serializeProperties(callCtx, internalProps)); principal.setEntityVersion(principal.getEntityVersion() + 1); - writeEntity(callCtx, ms, principal, true); + ms.writeEntity(callCtx, principal, true); } return secrets; } @@ -1179,7 +1159,8 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str // persist this entity after changing it. This will update the version and update the last // updated time. Because the entity version is changed, we will update the change tracking table - PolarisBaseEntity persistedEntity = this.persistEntityAfterChange(callCtx, ms, entityRefreshed); + PolarisBaseEntity persistedEntity = + this.persistEntityAfterChange(callCtx, ms, entityRefreshed, false); return new EntityResult(persistedEntity); } @@ -1337,13 +1318,11 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str refreshEntityToRename.setParentId(resolver.getParentId()); } - // persist back to the active slice with its new name and parent - ms.writeToEntitiesActive(callCtx, refreshEntityToRename); - - // persist the entity after change. This wil update the lastUpdateTimestamp and bump up the - // version + // persist the entity after change. This will update the lastUpdateTimestamp and bump up the + // version. Indicate that the nameOrParent changed, so so that we also update any by-name + // lookups if applicable PolarisBaseEntity renamedEntityToReturn = - this.persistEntityAfterChange(callCtx, ms, refreshEntityToRename); + this.persistEntityAfterChange(callCtx, ms, refreshEntityToRename, true); return new EntityResult(renamedEntityToReturn); } @@ -1995,7 +1974,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( + 1)); task.setEntityVersion(task.getEntityVersion() + 1); task.setProperties(PolarisObjectMapperUtil.serializeProperties(callCtx, properties)); - writeEntity(callCtx, ms, task, false); + ms.writeEntity(callCtx, task, false); }); return new EntitiesResult(availableTasks); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index 883ac0f282..9eca18dfce 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -91,6 +91,10 @@ public PolarisBaseEntity lookupEntityByName( long parentId, int typeCode, @Nonnull String name) { + // TODO: Consistently pull down the runInTransaction logic without running into conflicting + // nested transactions into here so that instead of having the caller be responsible for + // initiating the runInReadTransaction, we make this method call inherently safe to do + // the two-phase lookup. PolarisEntitiesActiveKey entityActiveKey = new PolarisEntitiesActiveKey(catalogId, parentId, typeCode, name); @@ -148,6 +152,31 @@ protected abstract PolarisEntityActiveRecord lookupEntityActive( public abstract List lookupEntityActiveBatch( @Nonnull PolarisCallContext callCtx, List entityActiveKeys); + /** {@inheritDoc} */ + @Override + public void writeEntity( + @Nonnull PolarisCallContext callCtx, + @Nonnull PolarisBaseEntity entity, + boolean nameOrParentChanged) { + writeToEntities(callCtx, entity); + writeToEntitiesChangeTracking(callCtx, entity); + + if (nameOrParentChanged) { + writeToEntitiesActive(callCtx, entity); + } + } + + /** + * Write the base entity to the entities table. If there is a conflict (existing record with the + * same id), all attributes of the new record will replace the existing one. + * + * @param callCtx call context + * @param entity entity record to write, potentially replacing an existing entity record with the + * same key + */ + protected abstract void writeToEntities( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + /** * Write the base entity to the entities_active table. If there is a conflict (existing record * with the same PK), all attributes of the new record will replace the existing one. @@ -156,7 +185,7 @@ public abstract List lookupEntityActiveBatch( * @param entity entity record to write, potentially replacing an existing entity record with the * same key */ - public abstract void writeToEntitiesActive( + protected abstract void writeToEntitiesActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); /** @@ -167,7 +196,7 @@ public abstract void writeToEntitiesActive( * @param entity entity record to write, potentially replacing an existing entity record with the * same key */ - public abstract void writeToEntitiesChangeTracking( + protected abstract void writeToEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); /** From b739fd1a70a198951e659d40b025651773528271 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Wed, 26 Feb 2025 08:19:38 +0000 Subject: [PATCH 10/15] Add originalEntity to the writeEntity method to enable compare-and-swap behavior from the underlying BasePersistence. Pushdown all the deleteFromEntities* methods into PolarisMetaStoreSession and add deleteEntity to BasePersistence which encapsulates handling the separate slices. --- .../core/persistence/BasePersistence.java | 11 +++-- .../PolarisMetaStoreManagerImpl.java | 48 +++++++++++-------- .../persistence/PolarisMetaStoreSession.java | 32 +++++++++++-- 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java index 0e9e2fab89..1181033145 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -57,11 +57,14 @@ public interface BasePersistence { * @param callCtx call context * @param entity entity to persist * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntity original state of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity */ void writeEntity( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity, - boolean nameOrParentChanged); + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity); /** * Write the specified grantRecord to the grant_records table. If there is a conflict (existing @@ -75,12 +78,12 @@ void writeToGrantRecords( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisGrantRecord grantRec); /** - * Delete the base entity from the entities table. + * Delete this entity from the meta store. * * @param callCtx call context - * @param entity entity record to delete + * @param entity entity to delete */ - void deleteFromEntities(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); + void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); /** * Delete the specified grantRecord to the grant_records table. diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index cafb828ca3..8fa1b1851a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -129,7 +129,7 @@ private void persistNewEntity( entity.setToPurgeTimestamp(0); // write it - ms.writeEntity(callCtx, entity, true); + ms.writeEntity(callCtx, entity, true, null); } /** @@ -141,13 +141,15 @@ private void persistNewEntity( * @param ms meta store * @param entity the entity which has been changed * @param nameOrParentChanged indicates if parent or name changed + * @param originalEntity the original state of the entity before changes * @return the entity with its version and lastUpdateTimestamp updated */ private @Nonnull PolarisBaseEntity persistEntityAfterChange( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisMetaStoreSession ms, @Nonnull PolarisBaseEntity entity, - boolean nameOrParentChanged) { + boolean nameOrParentChanged, + @Nonnull PolarisBaseEntity originalEntity) { // validate the entity type and subtype callCtx.getDiagServices().checkNotNull(entity, "unexpected_null_entity"); @@ -190,7 +192,7 @@ private void persistNewEntity( entity.setEntityVersion(entity.getEntityVersion() + 1); // persist it to the various slices - ms.writeEntity(callCtx, entity, nameOrParentChanged); + ms.writeEntity(callCtx, entity, nameOrParentChanged, originalEntity); // return it return entity; @@ -222,9 +224,6 @@ private void dropEntity( // creation timestamp must be filled callCtx.getDiagServices().check(entity.getDropTimestamp() == 0, "already_dropped"); - // delete it from active slice - ms.deleteFromEntitiesActive(callCtx, entity); - // for now drop all associated grants, etc. synchronously // delete ALL grant records to (if the entity is a grantee) and from that entity final List grantsOnGrantee = @@ -252,13 +251,13 @@ private void dropEntity( List entities = ms.lookupEntities(callCtx, new ArrayList<>(entityIdsGrantChanged)); for (PolarisBaseEntity entityGrantChanged : entities) { + PolarisBaseEntity originalEntity = new PolarisBaseEntity(entityGrantChanged); entityGrantChanged.setGrantRecordsVersion(entityGrantChanged.getGrantRecordsVersion() + 1); - ms.writeEntity(callCtx, entityGrantChanged, false); + ms.writeEntity(callCtx, entityGrantChanged, false, originalEntity); } // remove the entity being dropped now - ms.deleteFromEntities(callCtx, entity); - ms.deleteFromEntitiesChangeTracking(callCtx, entity); + ms.deleteEntity(callCtx, entity); // if it is a principal, we also need to drop the secrets if (entity.getType() == PolarisEntityType.PRINCIPAL) { @@ -321,10 +320,11 @@ private void dropEntity( callCtx .getDiagServices() .checkNotNull(granteeEntity, "grantee_not_found", "grantee={}", grantee); + PolarisBaseEntity originalGranteeEntity = new PolarisBaseEntity(granteeEntity); // grants have changed, we need to bump-up the grants version granteeEntity.setGrantRecordsVersion(granteeEntity.getGrantRecordsVersion() + 1); - ms.writeEntity(callCtx, granteeEntity, false); + ms.writeEntity(callCtx, granteeEntity, false, originalGranteeEntity); // we also need to invalidate the grants on that securable so that we can reload them. // load the securable and increment its grants version @@ -333,10 +333,11 @@ private void dropEntity( callCtx .getDiagServices() .checkNotNull(securableEntity, "securable_not_found", "securable={}", securable); + PolarisBaseEntity originalSecurableEntity = new PolarisBaseEntity(securableEntity); // grants have changed, we need to bump-up the grants version securableEntity.setGrantRecordsVersion(securableEntity.getGrantRecordsVersion() + 1); - ms.writeEntity(callCtx, securableEntity, false); + ms.writeEntity(callCtx, securableEntity, false, originalSecurableEntity); // done, return the new grant record return grantRecord; @@ -396,10 +397,11 @@ private void revokeGrantRecord( .getDiagServices() .checkNotNull( refreshGrantee, "missing_grantee", "grantRecord={} grantee={}", grantRecord, grantee); + PolarisBaseEntity originalRefreshGrantee = new PolarisBaseEntity(refreshGrantee); // grants have changed, we need to bump-up the grants version refreshGrantee.setGrantRecordsVersion(refreshGrantee.getGrantRecordsVersion() + 1); - ms.writeEntity(callCtx, refreshGrantee, false); + ms.writeEntity(callCtx, refreshGrantee, false, originalRefreshGrantee); // we also need to invalidate the grants on that securable so that we can reload them. // load the securable and increment its grants version @@ -413,10 +415,11 @@ private void revokeGrantRecord( "grantRecord={} securable={}", grantRecord, securable); + PolarisBaseEntity originalRefreshSecurable = new PolarisBaseEntity(refreshSecurable); // grants have changed, we need to bump-up the grants version refreshSecurable.setGrantRecordsVersion(refreshSecurable.getGrantRecordsVersion() + 1); - ms.writeEntity(callCtx, refreshSecurable, false); + ms.writeEntity(callCtx, refreshSecurable, false, originalRefreshSecurable); } /** @@ -942,6 +945,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } PolarisBaseEntity principal = loadEntityResult.getEntity(); + PolarisBaseEntity originalPrincipal = new PolarisBaseEntity(principal); Map internalProps = PolarisObjectMapperUtil.deserializeProperties( callCtx, @@ -963,14 +967,14 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str principal.setInternalProperties( PolarisObjectMapperUtil.serializeProperties(callCtx, internalProps)); principal.setEntityVersion(principal.getEntityVersion() + 1); - ms.writeEntity(callCtx, principal, true); + ms.writeEntity(callCtx, principal, true, originalPrincipal); } else if (internalProps.containsKey( PolarisEntityConstants.PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE)) { internalProps.remove(PolarisEntityConstants.PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_STATE); principal.setInternalProperties( PolarisObjectMapperUtil.serializeProperties(callCtx, internalProps)); principal.setEntityVersion(principal.getEntityVersion() + 1); - ms.writeEntity(callCtx, principal, true); + ms.writeEntity(callCtx, principal, true, originalPrincipal); } return secrets; } @@ -1153,6 +1157,8 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str return new EntityResult(BaseResult.ReturnStatus.TARGET_ENTITY_CONCURRENTLY_MODIFIED, null); } + PolarisBaseEntity originalEntity = new PolarisBaseEntity(entityRefreshed); + // update the two properties entityRefreshed.setInternalProperties(entity.getInternalProperties()); entityRefreshed.setProperties(entity.getProperties()); @@ -1160,7 +1166,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str // persist this entity after changing it. This will update the version and update the last // updated time. Because the entity version is changed, we will update the change tracking table PolarisBaseEntity persistedEntity = - this.persistEntityAfterChange(callCtx, ms, entityRefreshed, false); + this.persistEntityAfterChange(callCtx, ms, entityRefreshed, false, originalEntity); return new EntityResult(persistedEntity); } @@ -1305,8 +1311,9 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS, entityActiveRecord.getSubTypeCode()); } - // all good, delete the existing entity from the active slice - ms.deleteFromEntitiesActive(callCtx, refreshEntityToRename); + // Create a copy of the original before we change its fields so that we can pass in the + // old version for the persistence layer to work out whether to unlink previous name-lookups + PolarisBaseEntity originalEntity = new PolarisBaseEntity(refreshEntityToRename); // change its name now refreshEntityToRename.setName(renamedEntity.getName()); @@ -1322,7 +1329,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str // version. Indicate that the nameOrParent changed, so so that we also update any by-name // lookups if applicable PolarisBaseEntity renamedEntityToReturn = - this.persistEntityAfterChange(callCtx, ms, refreshEntityToRename, true); + this.persistEntityAfterChange(callCtx, ms, refreshEntityToRename, true, originalEntity); return new EntityResult(renamedEntityToReturn); } @@ -1961,6 +1968,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( availableTasks.forEach( task -> { + PolarisBaseEntity originalTask = new PolarisBaseEntity(task); Map properties = PolarisObjectMapperUtil.deserializeProperties(callCtx, task.getProperties()); properties.put(PolarisTaskConstants.LAST_ATTEMPT_EXECUTOR_ID, executorId); @@ -1974,7 +1982,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( + 1)); task.setEntityVersion(task.getEntityVersion() + 1); task.setProperties(PolarisObjectMapperUtil.serializeProperties(callCtx, properties)); - ms.writeEntity(callCtx, task, false); + ms.writeEntity(callCtx, task, false, originalTask); }); return new EntitiesResult(availableTasks); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index 9eca18dfce..f18b57ae70 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -157,11 +157,20 @@ public abstract List lookupEntityActiveBatch( public void writeEntity( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity, - boolean nameOrParentChanged) { + boolean nameOrParentChanged, + @Nullable PolarisBaseEntity originalEntity) { + // TODO: Pull down relevant compare-and-swap semantics from PolarisMetaStoreManagerImpl + // into this layer. writeToEntities(callCtx, entity); writeToEntitiesChangeTracking(callCtx, entity); if (nameOrParentChanged) { + if (originalEntity != null) { + // In our case, rename isn't automatically handled when the main "entities" slice + // is updated; instead we must explicitly remove from the old entitiesActive + // key as well. + deleteFromEntitiesActive(callCtx, originalEntity); + } writeToEntitiesActive(callCtx, entity); } } @@ -199,13 +208,30 @@ protected abstract void writeToEntitiesActive( protected abstract void writeToEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity); + /** {@inheritDoc} */ + @Override + public void deleteEntity(@Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity entity) { + deleteFromEntitiesActive(callCtx, entity); + deleteFromEntities(callCtx, entity); + deleteFromEntitiesChangeTracking(callCtx, entity); + } + + /** + * Delete the base entity from the entities table. + * + * @param callCtx call context + * @param entity entity record to delete + */ + protected abstract void deleteFromEntities( + @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); + /** * Delete the base entity from the entities_active table. * * @param callCtx call context * @param entity entity record to delete */ - public abstract void deleteFromEntitiesActive( + protected abstract void deleteFromEntitiesActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); /** @@ -214,7 +240,7 @@ public abstract void deleteFromEntitiesActive( * @param callCtx call context * @param entity entity record to delete */ - public abstract void deleteFromEntitiesChangeTracking( + protected abstract void deleteFromEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntityCore entity); /** Rollback the current transaction */ From b52a4af3900be17bca0185ec7e8c8aed3aac3a10 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Wed, 26 Feb 2025 23:08:37 +0000 Subject: [PATCH 11/15] Break out external-integration related methods from BasePersistence into a new IntegrationPersistence interface; these methods encapsulate certain type-specific behaviors that are indirectly tied to persistence entities, such as principal secrets, storage integrations, etc. --- .../core/persistence/BasePersistence.java | 94 ------------- .../persistence/IntegrationPersistence.java | 129 ++++++++++++++++++ .../persistence/PolarisMetaStoreSession.java | 2 +- 3 files changed, 130 insertions(+), 95 deletions(-) create mode 100644 polaris-core/src/main/java/org/apache/polaris/core/persistence/IntegrationPersistence.java diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java index 1181033145..339dfa1327 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -31,9 +31,6 @@ import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; -import org.apache.polaris.core.storage.PolarisStorageIntegration; /** * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like @@ -326,97 +323,6 @@ List loadAllGrantRecordsOnSecurable( List loadAllGrantRecordsOnGrantee( @Nonnull PolarisCallContext callCtx, long granteeCatalogId, long granteeId); - /** - * Allows to retrieve to the secrets of a principal given its unique client id - * - * @param callCtx call context - * @param clientId principal client id - * @return the secrets - */ - @Nullable - PolarisPrincipalSecrets loadPrincipalSecrets( - @Nonnull PolarisCallContext callCtx, @Nonnull String clientId); - - /** - * generate and store a client id and associated secrets for a newly created principal entity - * - * @param callCtx call context - * @param principalName name of the principal - * @param principalId principal id - */ - @Nonnull - PolarisPrincipalSecrets generateNewPrincipalSecrets( - @Nonnull PolarisCallContext callCtx, @Nonnull String principalName, long principalId); - - /** - * Rotate the secrets of a principal entity, i.e. make the specified main secrets the secondary - * and generate a new main secret - * - * @param callCtx call context - * @param clientId principal client id - * @param principalId principal id - * @param reset true if the principal secrets should be disabled and replaced with a one-time - * password - * @param oldSecretHash the principal secret's old main secret hash - */ - @Nullable - PolarisPrincipalSecrets rotatePrincipalSecrets( - @Nonnull PolarisCallContext callCtx, - @Nonnull String clientId, - long principalId, - boolean reset, - @Nonnull String oldSecretHash); - - /** - * When dropping a principal, we also need to drop the secrets of that principal - * - * @param callCtx the call context - * @param clientId principal client id - * @param principalId the id of the principal whose secrets are dropped - */ - void deletePrincipalSecrets( - @Nonnull PolarisCallContext callCtx, @Nonnull String clientId, long principalId); - - /** - * Create an in-memory storage integration - * - * @param callCtx the polaris calllctx - * @param catalogId the catalog id - * @param entityId the entity id - * @param polarisStorageConfigurationInfo the storage configuration information - * @return a storage integration object - */ - @Nullable - PolarisStorageIntegration createStorageIntegration( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long entityId, - PolarisStorageConfigurationInfo polarisStorageConfigurationInfo); - - /** - * Persist a storage integration in the metastore - * - * @param callContext the polaris call context - * @param entity the entity of the object - * @param storageIntegration the storage integration to persist - */ - void persistStorageIntegrationIfNeeded( - @Nonnull PolarisCallContext callContext, - @Nonnull PolarisBaseEntity entity, - @Nullable PolarisStorageIntegration storageIntegration); - - /** - * Load the polaris storage integration for a polaris entity (Catalog,Namespace,Table,View) - * - * @param callContext the polaris call context - * @param entity the polaris entity - * @return a polaris storage integration - */ - @Nullable - - PolarisStorageIntegration loadPolarisStorageIntegration( - @Nonnull PolarisCallContext callContext, @Nonnull PolarisBaseEntity entity); - /** * Check if the specified parent entity has children. * diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/IntegrationPersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/IntegrationPersistence.java new file mode 100644 index 0000000000..92a66dbfa6 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/IntegrationPersistence.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.persistence; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; + +/** + * Interface for the necessary "peripheral integration" objects that are logically attached to core + * persistence entities but which typically involve additional separate external integrations + * related to identity/auth, kms/secrets storage, etc. + * + *

Implementations should orchestrate any necessary multi-phase protocols such as leasing an + * external resource before committing a reference to the external resource in the Polaris + * persistence layer, etc. + */ +public interface IntegrationPersistence { + /** + * Allows to retrieve to the secrets of a principal given its unique client id + * + * @param callCtx call context + * @param clientId principal client id + * @return the secrets + */ + @Nullable + PolarisPrincipalSecrets loadPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId); + + /** + * generate and store a client id and associated secrets for a newly created principal entity + * + * @param callCtx call context + * @param principalName name of the principal + * @param principalId principal id + */ + @Nonnull + PolarisPrincipalSecrets generateNewPrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String principalName, long principalId); + + /** + * Rotate the secrets of a principal entity, i.e. make the specified main secrets the secondary + * and generate a new main secret + * + * @param callCtx call context + * @param clientId principal client id + * @param principalId principal id + * @param reset true if the principal secrets should be disabled and replaced with a one-time + * password + * @param oldSecretHash the principal secret's old main secret hash + */ + @Nullable + PolarisPrincipalSecrets rotatePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, + @Nonnull String clientId, + long principalId, + boolean reset, + @Nonnull String oldSecretHash); + + /** + * When dropping a principal, we also need to drop the secrets of that principal + * + * @param callCtx the call context + * @param clientId principal client id + * @param principalId the id of the principal whose secrets are dropped + */ + void deletePrincipalSecrets( + @Nonnull PolarisCallContext callCtx, @Nonnull String clientId, long principalId); + + /** + * Create an in-memory storage integration + * + * @param callCtx the polaris calllctx + * @param catalogId the catalog id + * @param entityId the entity id + * @param polarisStorageConfigurationInfo the storage configuration information + * @return a storage integration object + */ + @Nullable + PolarisStorageIntegration createStorageIntegration( + @Nonnull PolarisCallContext callCtx, + long catalogId, + long entityId, + PolarisStorageConfigurationInfo polarisStorageConfigurationInfo); + + /** + * Persist a storage integration in the metastore + * + * @param callContext the polaris call context + * @param entity the entity of the object + * @param storageIntegration the storage integration to persist + */ + void persistStorageIntegrationIfNeeded( + @Nonnull PolarisCallContext callContext, + @Nonnull PolarisBaseEntity entity, + @Nullable PolarisStorageIntegration storageIntegration); + + /** + * Load the polaris storage integration for a polaris entity (Catalog,Namespace,Table,View) + * + * @param callContext the polaris call context + * @param entity the polaris entity + * @return a polaris storage integration + */ + @Nullable + + PolarisStorageIntegration loadPolarisStorageIntegration( + @Nonnull PolarisCallContext callContext, @Nonnull PolarisBaseEntity entity); +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index f18b57ae70..c2b9665fed 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -33,7 +33,7 @@ * which can support a runInTransaction semantic, while providing default implementations of some of * the BasePersistence methods in terms of lower-level methods that subclasses must implement. */ -public abstract class PolarisMetaStoreSession implements BasePersistence { +public abstract class PolarisMetaStoreSession implements BasePersistence, IntegrationPersistence { /** * Run the specified transaction code (a Supplier lambda type) in a database read/write From f9c1fd71627863ba2415ac363702cfd611fe683a Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Thu, 27 Feb 2025 03:32:58 +0000 Subject: [PATCH 12/15] Improve javadoc comments, rename PolarisEntityActiveRecord to EntityNameLookupRecord, remove unused method --- ...olarisEclipseLinkMetaStoreSessionImpl.java | 26 ++--- .../eclipselink/PolarisEclipseLinkStore.java | 4 +- .../polaris/jpa/models/ModelEntityActive.java | 8 +- ...ecord.java => EntityNameLookupRecord.java} | 10 +- .../polaris/core/entity/PolarisEntity.java | 2 +- .../core/persistence/BasePersistence.java | 97 ++++++++++++------- .../persistence/PolarisEntityResolver.java | 6 +- .../persistence/PolarisMetaStoreManager.java | 10 +- .../PolarisMetaStoreManagerImpl.java | 15 ++- .../persistence/PolarisMetaStoreSession.java | 19 +++- .../PolarisTreeMapMetaStoreSessionImpl.java | 32 ++---- .../BasePolarisMetaStoreManagerTest.java | 8 +- .../PolarisTestMetaStoreManager.java | 16 +-- 13 files changed, 133 insertions(+), 120 deletions(-) rename polaris-core/src/main/java/org/apache/polaris/core/entity/{PolarisEntityActiveRecord.java => EntityNameLookupRecord.java} (92%) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 7de38105a6..6dce424740 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -39,10 +39,10 @@ import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; @@ -355,14 +355,6 @@ public void deleteAll(@Nonnull PolarisCallContext callCtx) { .toList(); } - /** {@inheritDoc} */ - @Override - public int lookupEntityVersion( - @Nonnull PolarisCallContext callCtx, long catalogId, long entityId) { - ModelEntity model = this.store.lookupEntity(localSession.get(), catalogId, entityId); - return model == null ? 0 : model.getEntityVersion(); - } - /** {@inheritDoc} */ @Override public @Nonnull List lookupEntityVersions( @@ -388,7 +380,7 @@ public int lookupEntityVersion( /** {@inheritDoc} */ @Override @Nullable - public PolarisEntityActiveRecord lookupEntityActive( + public EntityNameLookupRecord lookupEntityActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey) { // lookup the active entity slice return ModelEntityActive.toEntityActive( @@ -398,7 +390,7 @@ public PolarisEntityActiveRecord lookupEntityActive( /** {@inheritDoc} */ @Override @Nonnull - public List lookupEntityActiveBatch( + public List lookupEntityActiveBatch( @Nonnull PolarisCallContext callCtx, @Nonnull List entityActiveKeys) { // now build a list to quickly verify that nothing has changed @@ -409,23 +401,23 @@ public List lookupEntityActiveBatch( /** {@inheritDoc} */ @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType) { - return listActiveEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); + return listEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); } @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType, @Nonnull Predicate entityFilter) { // full range scan under the parent for that type - return listActiveEntities( + return listEntities( callCtx, catalogId, parentId, @@ -433,7 +425,7 @@ public List lookupEntityActiveBatch( Integer.MAX_VALUE, entityFilter, entity -> - new PolarisEntityActiveRecord( + new EntityNameLookupRecord( entity.getCatalogId(), entity.getId(), entity.getParentId(), @@ -443,7 +435,7 @@ public List lookupEntityActiveBatch( } @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index 59ac63f690..bfc83ae373 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -27,9 +27,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; @@ -99,7 +99,7 @@ void writeToEntitiesActive(EntityManager session, PolarisBaseEntity entity) { ModelEntityActive model = lookupEntityActive(session, new PolarisEntitiesActiveKey(entity)); if (model == null) { - session.persist(ModelEntityActive.fromEntityActive(new PolarisEntityActiveRecord(entity))); + session.persist(ModelEntityActive.fromEntityActive(new EntityNameLookupRecord(entity))); } } diff --git a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityActive.java b/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityActive.java index 2dab162ee1..56770d1be2 100644 --- a/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityActive.java +++ b/extension/persistence/jpa-model/src/main/java/org/apache/polaris/jpa/models/ModelEntityActive.java @@ -21,7 +21,7 @@ import jakarta.persistence.Entity; import jakarta.persistence.Id; import jakarta.persistence.Table; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; @@ -128,7 +128,7 @@ public ModelEntityActive build() { } } - public static ModelEntityActive fromEntityActive(PolarisEntityActiveRecord record) { + public static ModelEntityActive fromEntityActive(EntityNameLookupRecord record) { return ModelEntityActive.builder() .catalogId(record.getCatalogId()) .id(record.getId()) @@ -139,12 +139,12 @@ public static ModelEntityActive fromEntityActive(PolarisEntityActiveRecord recor .build(); } - public static PolarisEntityActiveRecord toEntityActive(ModelEntityActive model) { + public static EntityNameLookupRecord toEntityActive(ModelEntityActive model) { if (model == null) { return null; } - return new PolarisEntityActiveRecord( + return new EntityNameLookupRecord( model.catalogId, model.id, model.parentId, model.name, model.typeCode, model.subTypeCode); } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityActiveRecord.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/EntityNameLookupRecord.java similarity index 92% rename from polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityActiveRecord.java rename to polaris-core/src/main/java/org/apache/polaris/core/entity/EntityNameLookupRecord.java index 87682f4cb2..6d150a50f0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityActiveRecord.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/EntityNameLookupRecord.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; -public class PolarisEntityActiveRecord { +public class EntityNameLookupRecord { // entity catalog id private final long catalogId; @@ -74,7 +74,7 @@ public PolarisEntitySubType getSubType() { } @JsonCreator - public PolarisEntityActiveRecord( + public EntityNameLookupRecord( @JsonProperty("catalogId") long catalogId, @JsonProperty("id") long id, @JsonProperty("parentId") long parentId, @@ -90,7 +90,7 @@ public PolarisEntityActiveRecord( } /** Constructor to create the object with provided entity */ - public PolarisEntityActiveRecord(PolarisBaseEntity entity) { + public EntityNameLookupRecord(PolarisBaseEntity entity) { this.catalogId = entity.getCatalogId(); this.id = entity.getId(); this.parentId = entity.getParentId(); @@ -102,8 +102,8 @@ public PolarisEntityActiveRecord(PolarisBaseEntity entity) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof PolarisEntityActiveRecord)) return false; - PolarisEntityActiveRecord that = (PolarisEntityActiveRecord) o; + if (!(o instanceof EntityNameLookupRecord)) return false; + EntityNameLookupRecord that = (EntityNameLookupRecord) o; return catalogId == that.catalogId && id == that.id && parentId == that.parentId diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java index 3ded968af2..c68d1c974d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java @@ -172,7 +172,7 @@ public static List toCoreList(List path) { .orElse(null); } - public static List toNameAndIdList(List entities) { + public static List toNameAndIdList(List entities) { return Optional.ofNullable(entities) .map( list -> diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java index 339dfa1327..31c2d502d8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -24,18 +24,18 @@ import java.util.function.Function; import java.util.function.Predicate; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; /** - * Interface to the Polaris metadata store, allows to persist and retrieve all Polaris metadata like - * metadata for Polaris entities and metadata about grants between these entities which is the - * foundation of our role base access control model. + * Interface to the Polaris persistence backend, with which to persist and retrieve all the data + * defining the internal data model for Polaris, and which defines the basis for the RBAC model + * provided by Polaris. * *

Note that APIs to the actual persistence store are very basic, often point read or write to * the underlying data store. The goal is to make it really easy to back this using databases like @@ -43,13 +43,22 @@ */ public interface BasePersistence { /** + * The returned id must be fully unique within a realm and never reused once generated, whether or + * not anything ends up committing an entity with the generated id. + * * @param callCtx call context * @return new unique entity identifier */ long generateNewId(@Nonnull PolarisCallContext callCtx); /** - * Write this entity to the meta store. + * Write this entity to the persistence backend. If successful, the write must be durable and + * visible to any other reader. + * + *

TODO: Either standardize the expected system of exceptions to throw for various concurrency + * errors (entity not found when originalEntity != null, entity changed from originalEntity, etc) + * or push down the return status enums from PolarisMetaStoreManager into this layer and document + * accordingly. * * @param callCtx call context * @param entity entity to persist @@ -63,6 +72,27 @@ void writeEntity( boolean nameOrParentChanged, @Nullable PolarisBaseEntity originalEntity); + /** + * Atomically write a batch of entities to the persistence backend conditional on *every* member + * of originalEntities matching the existing persistent state. After this commit, *every* member + * of entities must be committed durably. + * + *

TODO: Push down the multi-entity commit from PolarisMetaStoreManagerImpl to use this instead + * of running single writeEntity actions within a transaction. + * + * @param callCtx call context + * @param entities entities to persist + * @param nameOrParentChanged if true, also write it to by-name lookups if applicable + * @param originalEntities original states of the entity to use for compare-and-swap purposes, or + * null if this is expected to be a brand-new entity; must contain exactly as many elements as + * {@code entities} where each item corresponds to the element of {@code entities} in the same + * index as this list. + */ + void writeEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List entities, + @Nullable List originalEntities); + /** * Write the specified grantRecord to the grant_records table. If there is a conflict (existing * record with the same PK), all attributes of the new record will replace the existing one. @@ -109,21 +139,22 @@ void deleteAllEntityGrantRecords( @Nonnull List grantsOnSecurable); /** - * Delete Polaris entity and grant record metadata from all tables. This is used during metadata - * bootstrap to reset all tables to their original state + * Delete Polaris entity and grant record metadata from all tables within the realm defined by the + * contents of the {@code callCtx} * * @param callCtx call context */ void deleteAll(@Nonnull PolarisCallContext callCtx); /** - * Lookup an entity given its catalog id (which can be NULL_ID for top-level entities) and its - * unique id. + * Lookup an entity given its catalog id (which can be {@link + * org.apache.polaris.core.entity.PolarisEntityConstants#NULL_ID} for top-level entities) and its + * entityId. * * @param callCtx call context * @param catalogId catalog id or NULL_ID - * @param entityId unique entity id - * @return NULL if the entity was not found, else the base entity. + * @param entityId entity id + * @return null if the entity was not found, else the retrieved entity. */ @Nullable PolarisBaseEntity lookupEntity( @@ -133,8 +164,12 @@ PolarisBaseEntity lookupEntity( * Lookup an entity given its catalogId, parentId, typeCode, and name. * * @param callCtx call context - * @param catalogId catalog id or NULL_ID - * @param parentId id of the parent, either a namespace or a catalog + * @param catalogId catalog id or {@link + * org.apache.polaris.core.entity.PolarisEntityConstants#NULL_ID} for top-level entities like + * CATALOG, PRINCIPAL and PRINCIPAL_ROLE. Note that by convention, a catalog itself has + * NULL_ID for its catalogId since the catalog is not "nested" under itself or any other + * catalog. + * @param parentId id of the parent * @param typeCode the PolarisEntityType code of the entity to lookup * @param name the name of the entity * @return null if the specified entity does not exist @@ -152,12 +187,12 @@ PolarisBaseEntity lookupEntityByName( * * @param callCtx call context * @param catalogId catalog id or NULL_ID - * @param parentId id of the parent, either a namespace or a catalog + * @param parentId id of the parent * @param typeCode the PolarisEntityType code of the entity to lookup * @param name the name of the entity * @return null if the specified entity does not exist */ - default PolarisEntityActiveRecord lookupEntityIdAndSubTypeByName( + default EntityNameLookupRecord lookupEntityIdAndSubTypeByName( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @@ -167,7 +202,7 @@ default PolarisEntityActiveRecord lookupEntityIdAndSubTypeByName( if (baseEntity == null) { return null; } - return new PolarisEntityActiveRecord(baseEntity); + return new EntityNameLookupRecord(baseEntity); } /** @@ -182,17 +217,6 @@ default PolarisEntityActiveRecord lookupEntityIdAndSubTypeByName( List lookupEntities( @Nonnull PolarisCallContext callCtx, List entityIds); - /** - * Lookup the current entityVersion of an entity given its catalog id (which can be NULL_ID for - * top-level entities) and its unique id. Will return 0 if the entity does not exist. - * - * @param callCtx call context - * @param catalogId catalog id or NULL_ID - * @param entityId unique entity id - * @return current version for that entity or 0 if entity was not found. - */ - int lookupEntityVersion(@Nonnull PolarisCallContext callCtx, long catalogId, long entityId); - /** * Get change tracking versions for all specified entity ids. * @@ -206,23 +230,23 @@ List lookupEntityVersions( @Nonnull PolarisCallContext callCtx, List entityIds); /** - * List all active entities of the specified type which are child entities of the specified parent + * List all entities of the specified type which are child entities of the specified parent * * @param callCtx call context * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level * @param parentId id of the parent, can be the special 0 value representing the root entity * @param entityType type of entities to list - * @return the list of entities_active records for the specified list operation + * @return the list of entities for the specified list operation */ @Nonnull - List listActiveEntities( + List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType); /** - * List active entities where some predicate returns true + * List entities where some predicate returns true * * @param callCtx call context * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level @@ -233,7 +257,7 @@ List listActiveEntities( * @return the list of entities for which the predicate returns true */ @Nonnull - List listActiveEntities( + List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @@ -241,8 +265,7 @@ List listActiveEntities( @Nonnull Predicate entityFilter); /** - * List active entities where some predicate returns true and transform the entities with a - * function + * List entities where some predicate returns true and transform the entities with a function * * @param callCtx call context * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level @@ -256,7 +279,7 @@ List listActiveEntities( * @return the list of entities for which the predicate returns true */ @Nonnull - List listActiveEntities( + List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @@ -326,11 +349,13 @@ List loadAllGrantRecordsOnGrantee( /** * Check if the specified parent entity has children. * + *

TODO: Figure out if this is needed vs listEntities with limit. + * * @param callContext the polaris call context * @param optionalEntityType if not null, only check for the specified type, else check for all * types of children entities * @param catalogId id of the catalog - * @param parentId id of the parent, either a namespace or a catalog + * @param parentId id of the parent * @return true if the parent entity has children */ boolean hasChildren( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java index 3d795d7359..fb321ff557 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java @@ -26,9 +26,9 @@ import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityType; @@ -282,13 +282,13 @@ private boolean resolveEntitiesIfNeeded( .collect(Collectors.toList()); // now lookup all these entities by name - Iterator activeRecordIt = + Iterator activeRecordIt = ms.lookupEntityActiveBatch(callCtx, entityActiveKeys).iterator(); // now validate if there was a change and if yes, re-resolve again for (PolarisEntityCore resolveEntity : toResolve) { // get associate active record - PolarisEntityActiveRecord activeEntityRecord = activeRecordIt.next(); + EntityNameLookupRecord activeEntityRecord = activeRecordIt.next(); // if this entity has been dropped (null) or replaced (<> ids), then fail validation if (activeEntityRecord == null || activeEntityRecord.getId() != resolveEntity.getId()) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java index 30c72a26a1..8a364ef65b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java @@ -28,10 +28,10 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisGrantManager; import org.apache.polaris.core.auth.PolarisSecretsManager; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntitySubType; @@ -175,7 +175,7 @@ PolarisMetaStoreManager.EntityResult readEntityByName( class ListEntitiesResult extends BaseResult { // null if not success. Else the list of entities being returned - private final List entities; + private final List entities; /** * Constructor for an error @@ -194,7 +194,7 @@ public ListEntitiesResult( * * @param entities list of entities being returned, implies success */ - public ListEntitiesResult(@Nonnull List entities) { + public ListEntitiesResult(@Nonnull List entities) { super(ReturnStatus.SUCCESS); this.entities = entities; } @@ -203,12 +203,12 @@ public ListEntitiesResult(@Nonnull List entities) { private ListEntitiesResult( @JsonProperty("returnStatus") @Nonnull ReturnStatus returnStatus, @JsonProperty("extraInformation") String extraInformation, - @JsonProperty("entities") List entities) { + @JsonProperty("entities") List entities) { super(returnStatus, extraInformation); this.entities = entities; } - public List getEntities() { + public List getEntities() { return entities; } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 8fa1b1851a..816311984b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -37,10 +37,10 @@ import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; @@ -721,9 +721,8 @@ private void bootstrapPolarisService( } // return list of active entities - List toreturnList = - ms.listActiveEntities( - callCtx, resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType); + List toreturnList = + ms.listEntities(callCtx, resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType); // prune the returned list with only entities matching the entity subtype if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { @@ -1065,7 +1064,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str } // check if an entity does not already exist with the same name. If true, this is an error - PolarisEntityActiveRecord entityActiveRecord = + EntityNameLookupRecord entityActiveRecord = ms.lookupEntityIdAndSubTypeByName( callCtx, entity.getCatalogId(), @@ -1299,7 +1298,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str // ensure that nothing exists where we create that entity // if this entity already exists, this is an error - PolarisEntityActiveRecord entityActiveRecord = + EntityNameLookupRecord entityActiveRecord = ms.lookupEntityIdAndSubTypeByName( callCtx, resolver.getCatalogIdOrNull(), @@ -1403,7 +1402,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str // get the list of catalog roles, at most 2 List catalogRoles = - ms.listActiveEntities( + ms.listEntities( callCtx, catalogId, catalogId, @@ -1944,7 +1943,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( // find all available tasks List availableTasks = - ms.listActiveEntities( + ms.listEntities( callCtx, PolarisEntityConstants.getRootEntityId(), PolarisEntityConstants.getRootEntityId(), diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java index c2b9665fed..9d3afac483 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java @@ -23,9 +23,9 @@ import java.util.List; import java.util.function.Supplier; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; /** @@ -83,6 +83,15 @@ public abstract T runInReadTransaction( public abstract void runActionInReadTransaction( @Nonnull PolarisCallContext callCtx, @Nonnull Runnable transactionCode); + /** {@inheritDoc} */ + @Override + public void writeEntities( + @Nonnull PolarisCallContext callCtx, + @Nonnull List entities, + @Nullable List originalEntities) { + throw new UnsupportedOperationException("Not yet implemented"); + } + /** {@inheritDoc} */ @Override public PolarisBaseEntity lookupEntityByName( @@ -99,7 +108,7 @@ public PolarisBaseEntity lookupEntityByName( new PolarisEntitiesActiveKey(catalogId, parentId, typeCode, name); // ensure that the entity exists - PolarisEntityActiveRecord entityActiveRecord = lookupEntityActive(callCtx, entityActiveKey); + EntityNameLookupRecord entityActiveRecord = lookupEntityActive(callCtx, entityActiveKey); // if not found, return null if (entityActiveRecord == null) { @@ -120,7 +129,7 @@ public PolarisBaseEntity lookupEntityByName( /** {@inheritDoc} */ @Override - public PolarisEntityActiveRecord lookupEntityIdAndSubTypeByName( + public EntityNameLookupRecord lookupEntityIdAndSubTypeByName( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @@ -139,7 +148,7 @@ public PolarisEntityActiveRecord lookupEntityIdAndSubTypeByName( * @return null if the specified entity does not exist or has been dropped. */ @Nullable - protected abstract PolarisEntityActiveRecord lookupEntityActive( + protected abstract EntityNameLookupRecord lookupEntityActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey); /** @@ -149,7 +158,7 @@ protected abstract PolarisEntityActiveRecord lookupEntityActive( * @return the list of entityActiveKeys for the specified lookup operation */ @Nonnull - public abstract List lookupEntityActiveBatch( + public abstract List lookupEntityActiveBatch( @Nonnull PolarisCallContext callCtx, List entityActiveKeys); /** {@inheritDoc} */ diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index 8e29bc248f..7a4315f3c8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -27,10 +27,10 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; import org.apache.polaris.core.entity.PolarisEntityType; @@ -230,18 +230,6 @@ public void deleteAll(@Nonnull PolarisCallContext callCtx) { .collect(Collectors.toList()); } - /** {@inheritDoc} */ - @Override - public int lookupEntityVersion( - @Nonnull PolarisCallContext callCtx, long catalogId, long entityId) { - PolarisBaseEntity baseEntity = - this.store - .getSliceEntitiesChangeTracking() - .read(this.store.buildKeyComposite(catalogId, entityId)); - - return baseEntity == null ? 0 : baseEntity.getEntityVersion(); - } - /** {@inheritDoc} */ @Override public @Nonnull List lookupEntityVersions( @@ -265,7 +253,7 @@ public int lookupEntityVersion( /** {@inheritDoc} */ @Override @Nullable - public PolarisEntityActiveRecord lookupEntityActive( + public EntityNameLookupRecord lookupEntityActive( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisEntitiesActiveKey entityActiveKey) { // lookup the active entity slice PolarisBaseEntity entity = @@ -281,7 +269,7 @@ public PolarisEntityActiveRecord lookupEntityActive( // return record return (entity == null) ? null - : new PolarisEntityActiveRecord( + : new EntityNameLookupRecord( entity.getCatalogId(), entity.getId(), entity.getParentId(), @@ -293,7 +281,7 @@ public PolarisEntityActiveRecord lookupEntityActive( /** {@inheritDoc} */ @Override @Nonnull - public List lookupEntityActiveBatch( + public List lookupEntityActiveBatch( @Nonnull PolarisCallContext callCtx, @Nonnull List entityActiveKeys) { // now build a list to quickly verify that nothing has changed @@ -304,23 +292,23 @@ public List lookupEntityActiveBatch( /** {@inheritDoc} */ @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType) { - return listActiveEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); + return listEntities(callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue()); } @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType, @Nonnull Predicate entityFilter) { // full range scan under the parent for that type - return listActiveEntities( + return listEntities( callCtx, catalogId, parentId, @@ -328,7 +316,7 @@ public List lookupEntityActiveBatch( Integer.MAX_VALUE, entityFilter, entity -> - new PolarisEntityActiveRecord( + new EntityNameLookupRecord( entity.getCatalogId(), entity.getId(), entity.getParentId(), @@ -338,7 +326,7 @@ public List lookupEntityActiveBatch( } @Override - public @Nonnull List listActiveEntities( + public @Nonnull List listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java index 1b5483afe5..34b325160c 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/BasePolarisMetaStoreManagerTest.java @@ -37,9 +37,9 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.AsyncTaskType; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.TaskEntity; @@ -122,7 +122,7 @@ void testCreateEntities() { .extracting(PolarisEntity::toCore) .containsExactly(PolarisEntity.toCore(task1), PolarisEntity.toCore(task2)); - List listedEntities = + List listedEntities = metaStoreManager .listEntities( polarisTestMetaStoreManager.polarisCallContext, @@ -134,14 +134,14 @@ void testCreateEntities() { .isNotNull() .hasSize(2) .containsExactly( - new PolarisEntityActiveRecord( + new EntityNameLookupRecord( task1.getCatalogId(), task1.getId(), task1.getParentId(), task1.getName(), task1.getTypeCode(), task1.getSubTypeCode()), - new PolarisEntityActiveRecord( + new EntityNameLookupRecord( task2.getCatalogId(), task2.getId(), task2.getParentId(), diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java index 0c7c4131da..5270990a9a 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java @@ -30,10 +30,10 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisGrantManager.LoadGrantsResult; +import org.apache.polaris.core.entity.EntityNameLookupRecord; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisChangeTrackingVersions; import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntityActiveRecord; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; @@ -657,7 +657,7 @@ void dropEntity(List catalogPath, PolarisEntityCore entityToD path.add(entityToDrop); // get all children, cannot be null - List children = + List children = polarisMetaStoreManager .listEntities( this.polarisCallContext, @@ -1189,7 +1189,7 @@ private void validateListReturn( List> expectedResult) { // list the entities under the specified path - List result = + List result = polarisMetaStoreManager .listEntities(this.polarisCallContext, path, entityType, entitySubType) .getEntities(); @@ -1201,7 +1201,7 @@ private void validateListReturn( // ensure all elements are found for (Pair expected : expectedResult) { boolean found = false; - for (PolarisEntityActiveRecord res : result) { + for (EntityNameLookupRecord res : result) { if (res.getName().equals(expected.getLeft()) && expected.getRight().getCode() == res.getSubTypeCode()) { found = true; @@ -1498,7 +1498,7 @@ private void refreshCacheEntry( /** validate that the root catalog was properly constructed */ void validateBootstrap() { // load all principals - List principals = + List principals = polarisMetaStoreManager .listEntities( this.polarisCallContext, @@ -1511,7 +1511,7 @@ void validateBootstrap() { Assertions.assertThat(principals).isNotNull().hasSize(1); // get catalog list information - PolarisEntityActiveRecord principalListInfo = principals.get(0); + EntityNameLookupRecord principalListInfo = principals.get(0); // now make sure this principal was properly persisted PolarisBaseEntity principal = @@ -1524,7 +1524,7 @@ void validateBootstrap() { PolarisEntitySubType.NULL_SUBTYPE); // load all principal roles - List principalRoles = + List principalRoles = polarisMetaStoreManager .listEntities( this.polarisCallContext, @@ -1537,7 +1537,7 @@ void validateBootstrap() { Assertions.assertThat(principalRoles).isNotNull().hasSize(1); // get catalog list information - PolarisEntityActiveRecord roleListInfo = principalRoles.get(0); + EntityNameLookupRecord roleListInfo = principalRoles.get(0); // now make sure this principal role was properly persisted PolarisBaseEntity principalRole = From f0f06adc44985e099c9a7963b3f10d0773e4d69d Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Thu, 27 Feb 2025 03:51:26 +0000 Subject: [PATCH 13/15] Rename PolarisMetaStoreSession to TransactionalPersistence and move into a new package "transactional". --- ...pseLinkPolarisMetaStoreManagerFactory.java | 4 +- ...olarisEclipseLinkMetaStoreSessionImpl.java | 4 +- .../polaris/core/PolarisCallContext.java | 10 +- .../LocalPolarisMetaStoreManagerFactory.java | 9 +- .../persistence/MetaStoreManagerFactory.java | 3 +- .../persistence/PolarisEntityResolver.java | 11 +- .../PolarisMetaStoreManagerImpl.java | 121 +++++++++--------- .../PolarisTreeMapMetaStoreSessionImpl.java | 3 +- .../TransactionalPersistence.java} | 6 +- .../core/persistence/EntityCacheTest.java | 3 +- .../core/persistence/ResolverTest.java | 3 +- .../cache/StorageCredentialCacheTest.java | 4 +- .../quarkus/config/QuarkusProducers.java | 6 +- .../quarkus/admin/PolarisAuthzTestBase.java | 4 +- .../catalog/BasePolarisCatalogTest.java | 4 +- .../task/TableCleanupTaskHandlerTest.java | 10 +- .../test/PolarisIntegrationTestFixture.java | 4 +- .../catalog/IcebergCatalogAdapter.java | 6 +- .../context/DefaultCallContextResolver.java | 4 +- ...nMemoryPolarisMetaStoreManagerFactory.java | 6 +- .../apache/polaris/service/TestServices.java | 4 +- 21 files changed, 119 insertions(+), 110 deletions(-) rename polaris-core/src/main/java/org/apache/polaris/core/persistence/{PolarisMetaStoreSession.java => transactional/TransactionalPersistence.java} (97%) diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index c53f1140c0..492f61668d 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -28,8 +28,8 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; /** @@ -60,7 +60,7 @@ protected PolarisEclipseLinkStore createBackingStore(@Nonnull PolarisDiagnostics } @Override - protected PolarisMetaStoreSession createMetaStoreSession( + protected TransactionalPersistence createMetaStoreSession( @Nonnull PolarisEclipseLinkStore store, @Nonnull RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet, diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index 6dce424740..4954de2ce9 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -50,9 +50,9 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.exceptions.AlreadyExistsException; import org.apache.polaris.core.persistence.BaseMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; import org.apache.polaris.core.persistence.RetryOnConcurrencyException; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -68,7 +68,7 @@ * EclipseLink implementation of a Polaris metadata store supporting persisting and retrieving all * Polaris metadata from/to the configured database systems. */ -public class PolarisEclipseLinkMetaStoreSessionImpl extends PolarisMetaStoreSession { +public class PolarisEclipseLinkMetaStoreSessionImpl extends TransactionalPersistence { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisEclipseLinkMetaStoreSessionImpl.class); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java index abe598714f..c584dde55c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java @@ -21,7 +21,7 @@ import jakarta.annotation.Nonnull; import java.time.Clock; import java.time.ZoneId; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; /** * The Call context is allocated each time a new REST request is processed. It contains instances of @@ -30,7 +30,7 @@ public class PolarisCallContext { // meta store which is used to persist Polaris entity metadata - private final PolarisMetaStoreSession metaStore; + private final TransactionalPersistence metaStore; // diag services private final PolarisDiagnostics diagServices; @@ -40,7 +40,7 @@ public class PolarisCallContext { private final Clock clock; public PolarisCallContext( - @Nonnull PolarisMetaStoreSession metaStore, + @Nonnull TransactionalPersistence metaStore, @Nonnull PolarisDiagnostics diagServices, @Nonnull PolarisConfigurationStore configurationStore, @Nonnull Clock clock) { @@ -51,7 +51,7 @@ public PolarisCallContext( } public PolarisCallContext( - @Nonnull PolarisMetaStoreSession metaStore, @Nonnull PolarisDiagnostics diagServices) { + @Nonnull TransactionalPersistence metaStore, @Nonnull PolarisDiagnostics diagServices) { this.metaStore = metaStore; this.diagServices = diagServices; this.configurationStore = new PolarisConfigurationStore() {}; @@ -66,7 +66,7 @@ public static PolarisCallContext copyOf(PolarisCallContext original) { original.getClock()); } - public PolarisMetaStoreSession getMetaStore() { + public TransactionalPersistence getMetaStore() { return metaStore; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 8c7d3bb5a8..0a26caf745 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -37,6 +37,7 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +54,7 @@ public abstract class LocalPolarisMetaStoreManagerFactory final Map storageCredentialCacheMap = new HashMap<>(); final Map entityCacheMap = new HashMap<>(); final Map backingStoreMap = new HashMap<>(); - final Map> sessionSupplierMap = new HashMap<>(); + final Map> sessionSupplierMap = new HashMap<>(); protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); private static final Logger LOGGER = @@ -68,7 +69,7 @@ protected LocalPolarisMetaStoreManagerFactory(@Nonnull PolarisDiagnostics diagno protected abstract StoreType createBackingStore(@Nonnull PolarisDiagnostics diagnostics); - protected abstract PolarisMetaStoreSession createMetaStoreSession( + protected abstract TransactionalPersistence createMetaStoreSession( @Nonnull StoreType store, @Nonnull RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet, @@ -118,7 +119,7 @@ public synchronized Map bootstrapRealms( public void purgeRealms(List realms) { for (String realm : realms) { PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(() -> realm); - PolarisMetaStoreSession session = getOrCreateSessionSupplier(() -> realm).get(); + TransactionalPersistence session = getOrCreateSessionSupplier(() -> realm).get(); PolarisCallContext callContext = new PolarisCallContext(session, diagServices); metaStoreManager.purge(callContext); @@ -142,7 +143,7 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( } @Override - public synchronized Supplier getOrCreateSessionSupplier( + public synchronized Supplier getOrCreateSessionSupplier( RealmContext realmContext) { if (!sessionSupplierMap.containsKey(realmContext.getRealmIdentifier())) { initializeForRealm(realmContext, null); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index 1355715786..56338fb571 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.cache.StorageCredentialCache; /** Configuration interface for configuring the {@link PolarisMetaStoreManager}. */ @@ -32,7 +33,7 @@ public interface MetaStoreManagerFactory { PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext); - Supplier getOrCreateSessionSupplier(RealmContext realmContext); + Supplier getOrCreateSessionSupplier(RealmContext realmContext); StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java index fb321ff557..cf6d3e79a0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java @@ -32,6 +32,7 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; /** * Utility class used by the meta store manager to ensure that all entities which had been resolved @@ -78,7 +79,7 @@ public class PolarisEntityResolver { */ PolarisEntityResolver( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nullable PolarisEntityCore resolvedEntity, @Nullable List otherTopLevelEntities) { @@ -157,7 +158,7 @@ public class PolarisEntityResolver { */ PolarisEntityResolver( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath) { this(callCtx, ms, catalogPath, null, null); } @@ -173,7 +174,7 @@ public class PolarisEntityResolver { */ PolarisEntityResolver( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, PolarisEntityCore resolvedEntityDto) { this(callCtx, ms, catalogPath, resolvedEntityDto, null); @@ -190,7 +191,7 @@ public class PolarisEntityResolver { */ PolarisEntityResolver( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { this(callCtx, ms, catalogPath, new PolarisEntityCore(entity), null); @@ -239,7 +240,7 @@ long getCatalogIdOrNull() { */ private boolean resolveEntitiesIfNeeded( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nullable PolarisEntityCore resolvedEntity, @Nullable List otherTopLevelEntities) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java index 816311984b..30aa4ef717 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java @@ -50,6 +50,7 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; @@ -78,7 +79,7 @@ public class PolarisMetaStoreManagerImpl extends BaseMetaStoreManager { */ private void persistNewEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisBaseEntity entity) { // validate the entity type and subtype @@ -146,7 +147,7 @@ private void persistNewEntity( */ private @Nonnull PolarisBaseEntity persistEntityAfterChange( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisBaseEntity entity, boolean nameOrParentChanged, @Nonnull PolarisBaseEntity originalEntity) { @@ -214,7 +215,7 @@ private void persistNewEntity( */ private void dropEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisBaseEntity entity) { // validate the entity type and subtype @@ -286,7 +287,7 @@ private void dropEntity( */ private @Nonnull PolarisGrantRecord persistNewGrantRecord( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore securable, @Nonnull PolarisEntityCore grantee, @Nonnull PolarisPrivilege priv) { @@ -355,7 +356,7 @@ private void dropEntity( */ private void revokeGrantRecord( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore securable, @Nonnull PolarisEntityCore grantee, @Nonnull PolarisGrantRecord grantRecord) { @@ -439,7 +440,7 @@ private void revokeGrantRecord( */ private @Nonnull CreateCatalogResult createCatalog( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisBaseEntity catalog, @Nullable PolarisStorageIntegration integration, @Nonnull List principalRoles) { @@ -561,7 +562,7 @@ private void revokeGrantRecord( * @param ms meta store in read/write mode */ private void bootstrapPolarisService( - @Nonnull PolarisCallContext callCtx, @Nonnull PolarisMetaStoreSession ms) { + @Nonnull PolarisCallContext callCtx, @Nonnull TransactionalPersistence ms) { // Create a root container entity that can represent the securable for any top-level grants. PolarisBaseEntity rootContainer = @@ -623,7 +624,7 @@ private void bootstrapPolarisService( @Override public @Nonnull BaseResult bootstrapPolarisService(@Nonnull PolarisCallContext callCtx) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // run operation in a read/write transaction ms.runActionInTransaction(callCtx, () -> this.bootstrapPolarisService(callCtx, ms)); @@ -635,7 +636,7 @@ private void bootstrapPolarisService( @Override public @Nonnull BaseResult purge(@Nonnull PolarisCallContext callCtx) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // run operation in a read/write transaction LOGGER.warn("Deleting all metadata in the metastore..."); @@ -652,7 +653,7 @@ private void bootstrapPolarisService( */ private @Nonnull PolarisMetaStoreManager.EntityResult readEntityByName( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisEntityType entityType, @Nonnull PolarisEntitySubType entitySubType, @@ -696,7 +697,7 @@ private void bootstrapPolarisService( @Nonnull PolarisEntitySubType entitySubType, @Nonnull String name) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // run operation in a read/write transaction return ms.runInReadTransaction( @@ -708,7 +709,7 @@ private void bootstrapPolarisService( */ private @Nonnull ListEntitiesResult listEntities( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisEntityType entityType, @Nonnull PolarisEntitySubType entitySubType) { @@ -744,7 +745,7 @@ private void bootstrapPolarisService( @Nonnull PolarisEntityType entityType, @Nonnull PolarisEntitySubType entitySubType) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // run operation in a read transaction return ms.runInReadTransaction( @@ -755,7 +756,7 @@ private void bootstrapPolarisService( @Override public @Nonnull GenerateEntityIdResult generateNewEntityId(@Nonnull PolarisCallContext callCtx) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); return new GenerateEntityIdResult(ms.generateNewId(callCtx)); } @@ -803,7 +804,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** {@link #createPrincipal(PolarisCallContext, PolarisBaseEntity)} */ private @Nonnull CreatePrincipalResult createPrincipal( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisBaseEntity principal) { // validate input callCtx.getDiagServices().checkNotNull(principal, "unexpected_null_principal"); @@ -900,7 +901,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str public @Nonnull CreatePrincipalResult createPrincipal( @Nonnull PolarisCallContext callCtx, @Nonnull PolarisBaseEntity principal) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction(callCtx, () -> this.createPrincipal(callCtx, ms, principal)); @@ -908,7 +909,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** See {@link #loadPrincipalSecrets(PolarisCallContext, String)} */ private @Nullable PolarisPrincipalSecrets loadPrincipalSecrets( - @Nonnull PolarisCallContext callCtx, PolarisMetaStoreSession ms, @Nonnull String clientId) { + @Nonnull PolarisCallContext callCtx, TransactionalPersistence ms, @Nonnull String clientId) { return ms.loadPrincipalSecrets(callCtx, clientId); } @@ -917,7 +918,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str public @Nonnull PrincipalSecretsResult loadPrincipalSecrets( @Nonnull PolarisCallContext callCtx, @Nonnull String clientId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction PolarisPrincipalSecrets secrets = @@ -931,7 +932,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** See {@link #} */ private @Nullable PolarisPrincipalSecrets rotatePrincipalSecrets( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull String clientId, long principalId, boolean reset, @@ -987,7 +988,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str boolean reset, @Nonnull String oldSecretHash) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction PolarisPrincipalSecrets secrets = @@ -1009,7 +1010,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nonnull PolarisBaseEntity catalog, @Nonnull List principalRoles) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); Map internalProp = getInternalPropertyMap(callCtx, catalog); String integrationIdentifierOrId = @@ -1038,7 +1039,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** {@link #createEntityIfNotExists(PolarisCallContext, List, PolarisBaseEntity)} */ private @Nonnull EntityResult createEntityIfNotExists( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { @@ -1090,7 +1091,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1103,7 +1104,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable List catalogPath, @Nonnull List entities) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1130,7 +1131,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private @Nonnull EntityResult updateEntityPropertiesIfNotChanged( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { // entity cannot be null @@ -1176,7 +1177,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable List catalogPath, @Nonnull PolarisBaseEntity entity) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1186,7 +1187,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str /** See {@link #updateEntitiesPropertiesIfNotChanged(PolarisCallContext, List)} */ private @Nonnull EntitiesResult updateEntitiesPropertiesIfNotChanged( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull List entities) { // ensure that the entities list is not null callCtx.getDiagServices().checkNotNull(entities, "unexpected_null_entities"); @@ -1221,7 +1222,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str public @Nonnull EntitiesResult updateEntitiesPropertiesIfNotChanged( @Nonnull PolarisCallContext callCtx, @Nonnull List entities) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1234,7 +1235,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private @Nonnull EntityResult renameEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisEntityCore entityToRename, @Nullable List newCatalogPath, @@ -1341,7 +1342,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable List newCatalogPath, @Nonnull PolarisEntity renamedEntity) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1358,7 +1359,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private @Nonnull DropEntityResult dropEntityIfExists( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @Nonnull PolarisEntityCore entityToDrop, @Nullable Map cleanupProperties, @@ -1473,7 +1474,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str @Nullable Map cleanupProperties, boolean cleanup) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1497,7 +1498,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private @Nonnull PolarisEntityResolver resolveRoleToGranteeUsageGrant( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable PolarisEntityCore catalog, @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { @@ -1547,7 +1548,7 @@ public Map deserializeProperties(PolarisCallContext callCtx, Str */ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore grantee, @Nullable List catalogPath, @Nonnull PolarisEntityCore securable) { @@ -1574,7 +1575,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( */ private @Nonnull PrivilegeResult grantUsageOnRoleToGrantee( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable PolarisEntityCore catalog, @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { @@ -1611,7 +1612,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1624,7 +1625,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( */ private @Nonnull PrivilegeResult revokeUsageOnRoleFromGrantee( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nullable PolarisEntityCore catalog, @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { @@ -1673,7 +1674,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisEntityCore role, @Nonnull PolarisEntityCore grantee) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1686,7 +1687,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( */ private @Nonnull PrivilegeResult grantPrivilegeOnSecurableToRole( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore grantee, @Nullable List catalogPath, @Nonnull PolarisEntityCore securable, @@ -1716,7 +1717,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisEntityCore securable, @Nonnull PolarisPrivilege privilege) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1732,7 +1733,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( */ private @Nonnull PrivilegeResult revokePrivilegeOnSecurableFromRole( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull PolarisEntityCore grantee, @Nullable List catalogPath, @Nonnull PolarisEntityCore securable, @@ -1778,7 +1779,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull PolarisEntityCore securable, @Nonnull PolarisPrivilege privilege) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read/write transaction return ms.runInTransaction( @@ -1791,7 +1792,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** {@link #loadGrantsOnSecurable(PolarisCallContext, long, long)} */ private @Nonnull LoadGrantsResult loadGrantsOnSecurable( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long securableCatalogId, long securableId) { @@ -1831,7 +1832,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( public @Nonnull LoadGrantsResult loadGrantsOnSecurable( @Nonnull PolarisCallContext callCtx, long securableCatalogId, long securableId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -1841,7 +1842,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** {@link #loadGrantsToGrantee(PolarisCallContext, long, long)} */ public @Nonnull LoadGrantsResult loadGrantsToGrantee( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long granteeCatalogId, long granteeId) { @@ -1880,7 +1881,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( public @Nonnull LoadGrantsResult loadGrantsToGrantee( @Nonnull PolarisCallContext callCtx, long granteeCatalogId, long granteeId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -1890,7 +1891,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** {@link PolarisMetaStoreManager#loadEntitiesChangeTracking(PolarisCallContext, List)} */ private @Nonnull ChangeTrackingResult loadEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, @Nonnull List entityIds) { List changeTracking = ms.lookupEntityVersions(callCtx, entityIds); @@ -1902,7 +1903,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( public @Nonnull ChangeTrackingResult loadEntitiesChangeTracking( @Nonnull PolarisCallContext callCtx, @Nonnull List entityIds) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -1912,7 +1913,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** Refer to {@link #loadEntity(PolarisCallContext, long, long)} */ private @Nonnull EntityResult loadEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long entityCatalogId, long entityId) { // this is an easy one @@ -1927,7 +1928,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( public @Nonnull EntityResult loadEntity( @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -1937,7 +1938,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( /** Refer to {@link #loadTasks(PolarisCallContext, String, int)} */ private @Nonnull EntitiesResult loadTasks( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, String executorId, int limit) { @@ -1989,7 +1990,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Override public @Nonnull EntitiesResult loadTasks( @Nonnull PolarisCallContext callCtx, String executorId, int limit) { - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); return ms.runInTransaction(callCtx, () -> this.loadTasks(callCtx, ms, executorId, limit)); } @@ -2004,7 +2005,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull Set allowedWriteLocations) { // get meta store session we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); callCtx .getDiagServices() .check( @@ -2058,7 +2059,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( @Nonnull Set actions, @Nonnull Set locations) { // get meta store we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); callCtx .getDiagServices() .check( @@ -2120,7 +2121,7 @@ public Map getInternalPropertyMap( /** {@link #loadResolvedEntityById(PolarisCallContext, long, long)} */ private @Nonnull ResolvedEntityResult loadResolvedEntityById( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long entityCatalogId, long entityId) { @@ -2151,7 +2152,7 @@ public Map getInternalPropertyMap( public @Nonnull ResolvedEntityResult loadResolvedEntityById( @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( @@ -2161,7 +2162,7 @@ public Map getInternalPropertyMap( /** {@link #loadResolvedEntityById(PolarisCallContext, long, long)} */ private @Nonnull ResolvedEntityResult loadResolvedEntityByName( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, long entityCatalogId, long parentId, @Nonnull PolarisEntityType entityType, @@ -2201,7 +2202,7 @@ public Map getInternalPropertyMap( @Nonnull PolarisEntityType entityType, @Nonnull String entityName) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction ResolvedEntityResult result = @@ -2260,7 +2261,7 @@ public Map getInternalPropertyMap( /** {@inheritDoc} */ private @Nonnull ResolvedEntityResult refreshResolvedEntity( @Nonnull PolarisCallContext callCtx, - @Nonnull PolarisMetaStoreSession ms, + @Nonnull TransactionalPersistence ms, int entityVersion, int entityGrantRecordsVersion, @Nonnull PolarisEntityType entityType, @@ -2319,7 +2320,7 @@ public Map getInternalPropertyMap( long entityCatalogId, long entityId) { // get metastore we should be using - PolarisMetaStoreSession ms = callCtx.getMetaStore(); + TransactionalPersistence ms = callCtx.getMetaStore(); // need to run inside a read transaction return ms.runInReadTransaction( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java index 7a4315f3c8..2802bca3be 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java @@ -36,11 +36,12 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -public class PolarisTreeMapMetaStoreSessionImpl extends PolarisMetaStoreSession { +public class PolarisTreeMapMetaStoreSessionImpl extends TransactionalPersistence { // the TreeMap store to use private final PolarisTreeMapStore store; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java similarity index 97% rename from polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java rename to polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java index 9d3afac483..856fc79210 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.transactional; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; @@ -27,13 +27,15 @@ import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; import org.apache.polaris.core.entity.PolarisEntityCore; +import org.apache.polaris.core.persistence.BasePersistence; +import org.apache.polaris.core.persistence.IntegrationPersistence; /** * Extends BasePersistence to express a more "transaction-oriented" control flow for backing stores * which can support a runInTransaction semantic, while providing default implementations of some of * the BasePersistence methods in terms of lower-level methods that subclasses must implement. */ -public abstract class PolarisMetaStoreSession implements BasePersistence, IntegrationPersistence { +public abstract class TransactionalPersistence implements BasePersistence, IntegrationPersistence { /** * Run the specified transaction code (a Supplier lambda type) in a database read/write diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java index 8dc3c84aba..04cefc6be3 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java @@ -33,6 +33,7 @@ import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; import org.apache.polaris.core.persistence.cache.EntityCacheLookupResult; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -47,7 +48,7 @@ public class EntityCacheTest { private final PolarisTreeMapStore store; // to interact with the metastore - private final PolarisMetaStoreSession metaStore; + private final TransactionalPersistence metaStore; // polaris call context private final PolarisCallContext callCtx; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java index 5270a90e22..23702336e3 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java @@ -49,6 +49,7 @@ import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -63,7 +64,7 @@ public class ResolverTest { private final PolarisTreeMapStore store; // to interact with the metastore - private final PolarisMetaStoreSession metaStore; + private final TransactionalPersistence metaStore; // polaris call context private final PolarisCallContext callCtx; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index a4f7c6936d..ed0be12044 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -39,10 +39,10 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.BaseResult; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; import org.apache.polaris.core.persistence.PolarisTreeMapStore; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisCredentialVendor.ScopedCredentialsResult; import org.assertj.core.api.Assertions; @@ -66,7 +66,7 @@ public StorageCredentialCacheTest() { // the entity store, use treemap implementation PolarisTreeMapStore store = new PolarisTreeMapStore(diagServices); // to interact with the metastore - PolarisMetaStoreSession metaStore = + TransactionalPersistence metaStore = new PolarisTreeMapMetaStoreSessionImpl(store, Mockito.mock(), RANDOM_SECRETS); callCtx = new PolarisCallContext(metaStore, diagServices); metaStoreManager = Mockito.mock(PolarisMetaStoreManager.class); diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java index a3eb9b1929..cbe1cc2627 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusProducers.java @@ -44,8 +44,8 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.auth.Authenticator; import org.apache.polaris.service.auth.TokenBrokerFactory; @@ -111,7 +111,7 @@ public PolarisCallContext polarisCallContext( PolarisConfigurationStore configurationStore, MetaStoreManagerFactory metaStoreManagerFactory, Clock clock) { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); return new PolarisCallContext(metaStoreSession, diagServices, configurationStore, clock); } @@ -225,7 +225,7 @@ public PolarisMetaStoreManager polarisMetaStoreManager( @Produces @RequestScoped - public PolarisMetaStoreSession polarisMetaStoreSession( + public TransactionalPersistence polarisMetaStoreSession( RealmContext realmContext, MetaStoreManagerFactory metaStoreManagerFactory) { return metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index e6670b4087..e1176ce515 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -71,8 +71,8 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; @@ -177,7 +177,7 @@ public Map getConfigOverrides() { protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; protected PolarisMetaStoreManager metaStoreManager; - protected PolarisMetaStoreSession metaStoreSession; + protected TransactionalPersistence metaStoreSession; protected PolarisBaseEntity catalogEntity; protected PrincipalEntity principalEntity; protected CallContext callContext; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 370ede6623..49d976fe66 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -86,10 +86,10 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageIntegration; @@ -328,7 +328,7 @@ public PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmCon } @Override - public Supplier getOrCreateSessionSupplier( + public Supplier getOrCreateSessionSupplier( RealmContext realmContext) { return () -> polarisContext.getMetaStore(); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java index 06901b0f13..ad2d0f055c 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java @@ -49,8 +49,8 @@ import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; @@ -176,7 +176,7 @@ public void testTableCleanup() throws IOException { @Test public void testTableCleanupHandlesAlreadyDeletedMetadata() throws IOException { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); FileIO fileIO = new InMemoryFileIO() { @@ -230,7 +230,7 @@ public void close() { @Test public void testTableCleanupDuplicatesTasksIfFileStillExists() throws IOException { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); FileIO fileIO = new InMemoryFileIO() { @@ -346,7 +346,7 @@ public void close() { @Test public void testTableCleanupMultipleSnapshots() throws IOException { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); @@ -491,7 +491,7 @@ public void testTableCleanupMultipleSnapshots() throws IOException { @Test public void testTableCleanupMultipleMetadata() throws IOException { - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); FileIO fileIO = new InMemoryFileIO(); TableIdentifier tableIdentifier = TableIdentifier.of(Namespace.of("db1", "schema1"), "table1"); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java index 9058de0981..7c7b6af8ae 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java @@ -43,8 +43,8 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.quarkus.auth.TokenUtils; import org.junit.jupiter.api.TestInfo; @@ -106,7 +106,7 @@ private PolarisPrincipalSecrets fetchAdminSecrets() { helper.realmContextResolver.resolveRealmContext( baseUri.toString(), "GET", "/", Map.of(REALM_PROPERTY_KEY, realm)); - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = helper.metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); PolarisCallContext polarisContext = new PolarisCallContext( diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 5ec318693c..f2ba62add8 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -62,10 +62,10 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApiService; import org.apache.polaris.service.context.CallContextCatalogFactory; @@ -123,7 +123,7 @@ public class IcebergCatalogAdapter private final CallContextCatalogFactory catalogFactory; private final PolarisEntityManager entityManager; private final PolarisMetaStoreManager metaStoreManager; - private final PolarisMetaStoreSession session; + private final TransactionalPersistence session; private final PolarisConfigurationStore configurationStore; private final PolarisDiagnostics diagnostics; private final PolarisAuthorizer polarisAuthorizer; @@ -136,7 +136,7 @@ public IcebergCatalogAdapter( CallContextCatalogFactory catalogFactory, PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, - PolarisMetaStoreSession session, + TransactionalPersistence session, PolarisConfigurationStore configurationStore, PolarisDiagnostics diagnostics, PolarisAuthorizer polarisAuthorizer, diff --git a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java index 985e57d84f..14246692e3 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java @@ -29,7 +29,7 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +64,7 @@ public CallContext resolveCallContext( // pushed down for the metaStoreManagerFactory to inject Transactional-DB specific things // (including the MetaStoreSession" into the PolarisCallContext. The non-transactional // factories would then inject something else instead if needed. - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); PolarisCallContext polarisContext = new PolarisCallContext(metaStoreSession, diagnostics, configurationStore, clock); diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 8bf862affd..327ba9270d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -33,10 +33,10 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; import org.apache.polaris.core.persistence.PolarisTreeMapStore; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.service.context.RealmContextConfiguration; @@ -69,7 +69,7 @@ protected PolarisTreeMapStore createBackingStore(@Nonnull PolarisDiagnostics dia } @Override - protected PolarisMetaStoreSession createMetaStoreSession( + protected TransactionalPersistence createMetaStoreSession( @Nonnull PolarisTreeMapStore store, @Nonnull RealmContext realmContext, @Nullable RootCredentialsSet rootCredentialsSet, @@ -89,7 +89,7 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( } @Override - public synchronized Supplier getOrCreateSessionSupplier( + public synchronized Supplier getOrCreateSessionSupplier( RealmContext realmContext) { String realmId = realmContext.getRealmIdentifier(); if (!bootstrappedRealms.contains(realmId)) { diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 0f9109b7a5..6451656280 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -40,7 +40,7 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; import org.apache.polaris.service.catalog.DefaultIcebergCatalogPrefixParser; @@ -135,7 +135,7 @@ public TestServices build() { realmEntityManagerFactory.getOrCreateEntityManager(realmContext); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); - PolarisMetaStoreSession metaStoreSession = + TransactionalPersistence metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); CallContext callContext = new CallContext() { From ea0a90fecbb53ac546e12c54b70b0a769f97e674 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Thu, 27 Feb 2025 03:54:12 +0000 Subject: [PATCH 14/15] Also move the PolarisTreeMap* classes into the transactional package --- .../PolarisTreeMapMetaStoreSessionImpl.java | 5 +++-- .../persistence/{ => transactional}/PolarisTreeMapStore.java | 2 +- .../org/apache/polaris/core/persistence/EntityCacheTest.java | 2 ++ .../core/persistence/PolarisTreeMapMetaStoreManagerTest.java | 2 ++ .../org/apache/polaris/core/persistence/ResolverTest.java | 2 ++ .../core/storage/cache/StorageCredentialCacheTest.java | 4 ++-- .../persistence/InMemoryPolarisMetaStoreManagerFactory.java | 4 ++-- 7 files changed, 14 insertions(+), 7 deletions(-) rename polaris-core/src/main/java/org/apache/polaris/core/persistence/{ => transactional}/PolarisTreeMapMetaStoreSessionImpl.java (98%) rename polaris-core/src/main/java/org/apache/polaris/core/persistence/{ => transactional}/PolarisTreeMapStore.java (99%) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapMetaStoreSessionImpl.java similarity index 98% rename from polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java rename to polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapMetaStoreSessionImpl.java index 2802bca3be..312f3576b9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapMetaStoreSessionImpl.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.transactional; import com.google.common.base.Predicates; import jakarta.annotation.Nonnull; @@ -36,7 +36,8 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.persistence.BaseMetaStoreManager; +import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapStore.java similarity index 99% rename from polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java rename to polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapStore.java index 3d5f3d7f9c..fe80bdf23a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapStore.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisTreeMapStore.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.transactional; import jakarta.annotation.Nonnull; import java.util.ArrayList; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java index 04cefc6be3..4c5155b8fa 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java @@ -33,6 +33,8 @@ import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; import org.apache.polaris.core.persistence.cache.EntityCacheLookupResult; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java index e44b455777..2b228e5fcb 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java @@ -25,6 +25,8 @@ import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; import org.mockito.Mockito; public class PolarisTreeMapMetaStoreManagerTest extends BasePolarisMetaStoreManagerTest { diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java index 23702336e3..6db5a4f282 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java @@ -49,6 +49,8 @@ import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index ed0be12044..848367fe5c 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -40,8 +40,8 @@ import org.apache.polaris.core.persistence.BaseResult; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisObjectMapperUtil; -import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; -import org.apache.polaris.core.persistence.PolarisTreeMapStore; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisCredentialVendor.ScopedCredentialsResult; diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 327ba9270d..407d514cbe 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -33,9 +33,9 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; -import org.apache.polaris.core.persistence.PolarisTreeMapStore; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; +import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.service.context.RealmContextConfiguration; From f7d602c8ddd613bf7a5dc04cb6006e864a4c3e54 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Fri, 28 Feb 2025 07:48:27 +0000 Subject: [PATCH 15/15] Move PolarisMetaStoreManagerImpl into the "transactional" package per PR suggestion --- .../PolarisEclipseLinkMetaStoreManagerTest.java | 2 +- .../persistence/LocalPolarisMetaStoreManagerFactory.java | 1 + .../polaris/core/persistence/PolarisObjectMapperUtil.java | 8 ++++---- .../{ => transactional}/PolarisEntityResolver.java | 3 +-- .../{ => transactional}/PolarisMetaStoreManagerImpl.java | 8 ++++---- .../apache/polaris/core/persistence/EntityCacheTest.java | 1 + .../persistence/PolarisTreeMapMetaStoreManagerTest.java | 1 + .../org/apache/polaris/core/persistence/ResolverTest.java | 1 + 8 files changed, 14 insertions(+), 11 deletions(-) rename polaris-core/src/main/java/org/apache/polaris/core/persistence/{ => transactional}/PolarisEntityResolver.java (98%) rename polaris-core/src/main/java/org/apache/polaris/core/persistence/{ => transactional}/PolarisMetaStoreManagerImpl.java (99%) diff --git a/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java b/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java index cd9b085902..9415696dae 100644 --- a/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java +++ b/extension/persistence/eclipselink/src/test/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreManagerTest.java @@ -41,8 +41,8 @@ import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.persistence.BasePolarisMetaStoreManagerTest; -import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl; import org.apache.polaris.core.persistence.PolarisTestMetaStoreManager; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; import org.apache.polaris.jpa.models.ModelPrincipalSecrets; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 0a26caf745..0ea992ed38 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -37,6 +37,7 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.slf4j.Logger; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisObjectMapperUtil.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisObjectMapperUtil.java index a91d2bbab5..89b0c6429f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisObjectMapperUtil.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisObjectMapperUtil.java @@ -114,9 +114,9 @@ public static Map deserializeProperties( return retProperties; } - static class TaskExecutionState { - final String executor; - final long lastAttemptStartTime; + public static class TaskExecutionState { + public final String executor; + public final long lastAttemptStartTime; final int attemptCount; TaskExecutionState(String executor, long lastAttemptStartTime, int attemptCount) { @@ -146,7 +146,7 @@ public int getAttemptCount() { * @param entity entity * @return TaskExecutionState */ - static @Nullable TaskExecutionState parseTaskState(PolarisBaseEntity entity) { + public static @Nullable TaskExecutionState parseTaskState(PolarisBaseEntity entity) { JsonFactory jfactory = new JsonFactory(); try (JsonParser jParser = jfactory.createParser(entity.getProperties())) { String executorId = null; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisEntityResolver.java similarity index 98% rename from polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java rename to polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisEntityResolver.java index cf6d3e79a0..bb4e39591e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityResolver.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisEntityResolver.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.transactional; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; @@ -32,7 +32,6 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; /** * Utility class used by the meta store manager to ensure that all entities which had been resolved diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisMetaStoreManagerImpl.java similarity index 99% rename from polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java rename to polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisMetaStoreManagerImpl.java index 30aa4ef717..da2ff2e924 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/PolarisMetaStoreManagerImpl.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.persistence; +package org.apache.polaris.core.persistence.transactional; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -50,7 +50,7 @@ import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PolarisTaskConstants; -import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; +import org.apache.polaris.core.persistence.*; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageActions; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; @@ -651,7 +651,7 @@ private void bootstrapPolarisService( * See {@link #readEntityByName(PolarisCallContext, List, PolarisEntityType, PolarisEntitySubType, * String)} */ - private @Nonnull PolarisMetaStoreManager.EntityResult readEntityByName( + private @Nonnull EntityResult readEntityByName( @Nonnull PolarisCallContext callCtx, @Nonnull TransactionalPersistence ms, @Nullable List catalogPath, @@ -690,7 +690,7 @@ private void bootstrapPolarisService( /** {@inheritDoc} */ @Override - public @Nonnull PolarisMetaStoreManager.EntityResult readEntityByName( + public @Nonnull EntityResult readEntityByName( @Nonnull PolarisCallContext callCtx, @Nullable List catalogPath, @Nonnull PolarisEntityType entityType, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java index 4c5155b8fa..d6015160ee 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java @@ -33,6 +33,7 @@ import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; import org.apache.polaris.core.persistence.cache.EntityCacheLookupResult; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java index 2b228e5fcb..65f8080d91 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreManagerTest.java @@ -25,6 +25,7 @@ import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; import org.mockito.Mockito; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java index 6db5a4f282..c3e5f75327 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java @@ -49,6 +49,7 @@ import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.core.persistence.transactional.PolarisMetaStoreManagerImpl; import org.apache.polaris.core.persistence.transactional.PolarisTreeMapMetaStoreSessionImpl; import org.apache.polaris.core.persistence.transactional.PolarisTreeMapStore; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence;