diff --git a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java index cf2e9053b1..1d708cce6f 100644 --- a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java +++ b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java @@ -19,7 +19,6 @@ package org.apache.polaris.extension.persistence.impl.eclipselink; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Predicates; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import jakarta.persistence.EntityManager; @@ -49,6 +48,7 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; @@ -424,50 +424,13 @@ public List lookupEntityActiveBatchInCurrentTxn( .collect(Collectors.toList()); } - /** {@inheritDoc} */ - @Override - public @Nonnull Page listEntitiesInCurrentTxn( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull PageToken pageToken) { - return this.listEntitiesInCurrentTxn( - callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue(), pageToken); - } - - @Override - public @Nonnull Page listEntitiesInCurrentTxn( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull Predicate entityFilter, - @Nonnull PageToken pageToken) { - // full range scan under the parent for that type - return this.listEntitiesInCurrentTxn( - callCtx, - catalogId, - parentId, - entityType, - entityFilter, - entity -> - new EntityNameLookupRecord( - entity.getCatalogId(), - entity.getId(), - entity.getParentId(), - entity.getName(), - entity.getTypeCode(), - entity.getSubTypeCode()), - pageToken); - } - @Override public @Nonnull Page listEntitiesInCurrentTxn( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull Predicate entityFilter, @Nonnull Function transformer, @Nonnull PageToken pageToken) { @@ -475,7 +438,7 @@ public List lookupEntityActiveBatchInCurrentTxn( Stream data = this.store .lookupFullEntitiesActive( - localSession.get(), catalogId, parentId, entityType, pageToken) + localSession.get(), catalogId, parentId, entityType, entitySubType, pageToken) .stream() .map(ModelEntity::toEntity) .filter(entityFilter); diff --git a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java index 60251a7b65..223567073a 100644 --- a/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java +++ b/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java @@ -32,6 +32,7 @@ import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; @@ -289,6 +290,7 @@ List lookupFullEntitiesActive( long catalogId, long parentId, @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull PageToken pageToken) { diagnosticServices.check(session != null, "session_is_null"); checkInitialized(); @@ -298,6 +300,10 @@ List lookupFullEntitiesActive( "SELECT m from ModelEntity m where" + " m.catalogId=:catalogId and m.parentId=:parentId and m.typeCode=:typeCode"; + if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { + hql += " and m.subTypeCode=:subTypeCode"; + } + var entityIdToken = pageToken.valueAs(EntityIdToken.class); if (entityIdToken.isPresent()) { hql += " and m.id > :tokenId"; @@ -314,6 +320,10 @@ List lookupFullEntitiesActive( .setParameter("parentId", parentId) .setParameter("typeCode", entityType.getCode()); + if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { + query.setParameter("subTypeCode", entitySubType.getCode()); + } + if (entityIdToken.isPresent()) { long tokenId = entityIdToken.get().entityId(); query = query.setParameter("tokenId", tokenId); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index 76da5fa92a..1babeb03ee 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -44,6 +44,7 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; @@ -427,43 +428,28 @@ public Page listEntities( long catalogId, long parentId, @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull PageToken pageToken) { + // TODO: only fetch the properties required for creating an EntityNameLookupRecord return listEntities( callCtx, catalogId, parentId, entityType, + entitySubType, entity -> true, EntityNameLookupRecord::new, pageToken); } - @Nonnull - @Override - public Page listEntities( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull Predicate entityFilter, - @Nonnull PageToken pageToken) { - return listEntities( - callCtx, - catalogId, - parentId, - entityType, - entityFilter, - EntityNameLookupRecord::new, - pageToken); - } - @Nonnull @Override public Page listEntities( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, - PolarisEntityType entityType, + @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull Predicate entityFilter, @Nonnull Function transformer, @Nonnull PageToken pageToken) { @@ -479,6 +465,12 @@ public Page listEntities( realmId); Map whereGreater; + if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { + Map updatedWhereEquals = new HashMap<>(whereEquals); + updatedWhereEquals.put("sub_type_code", entitySubType.getCode()); + whereEquals = updatedWhereEquals; + } + // Limit can't be pushed down, due to client side filtering // absence of transaction. String orderByColumnName = null; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java index 0a09b20ea3..9c35a7805a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java @@ -31,7 +31,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; -import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; @@ -709,14 +708,8 @@ private void revokeGrantRecord( ? 0L : catalogPath.get(catalogPath.size() - 1).getId(); - // prune the returned list with only entities matching the entity subtype - Predicate filter = - entitySubType != PolarisEntitySubType.ANY_SUBTYPE - ? e -> e.getSubTypeCode() == entitySubType.getCode() - : entity -> true; - Page resultPage = - ms.listEntities(callCtx, catalogId, parentId, entityType, filter, pageToken); + ms.listEntities(callCtx, catalogId, parentId, entityType, entitySubType, pageToken); // TODO: Use post-validation to enforce consistent view against catalogPath. In the // meantime, happens-before ordering semantics aren't guaranteed during high-concurrency @@ -1183,6 +1176,7 @@ private void revokeGrantRecord( catalogId, catalogId, PolarisEntityType.CATALOG_ROLE, + PolarisEntitySubType.ANY_SUBTYPE, entity -> true, Function.identity(), PageToken.fromLimit(2)) @@ -1504,6 +1498,7 @@ private void revokeGrantRecord( PolarisEntityConstants.getRootEntityId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.TASK, + PolarisEntitySubType.ANY_SUBTYPE, entity -> { PolarisObjectMapperUtil.TaskExecutionState taskState = PolarisObjectMapperUtil.parseTaskState(entity); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java index 26ccd8de39..1e8d2abf81 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java @@ -32,6 +32,7 @@ import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.persistence.pagination.Page; @@ -275,6 +276,7 @@ List lookupEntityVersions( * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level * @param parentId id of the parent, can be the special 0 value representing the root entity * @param entityType type of entities to list + * @param entitySubType subType of entities to list (or ANY_SUBTYPE) * @param pageToken the token to start listing after * @return the list of entities for the specified list operation */ @@ -284,27 +286,7 @@ Page listEntities( long catalogId, long parentId, @Nonnull PolarisEntityType entityType, - @Nonnull PageToken pageToken); - - /** - * List entities where some predicate returns true - * - * @param callCtx call context - * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level - * @param parentId id of the parent, can be the special 0 value representing the root entity - * @param entityType type of entities to list - * @param entityFilter the filter to be applied to each entity. Only entities where the predicate - * returns true are returned in the list - * @param pageToken the token to start listing after - * @return the list of entities for which the predicate returns true - */ - @Nonnull - Page listEntities( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull Predicate entityFilter, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull PageToken pageToken); /** @@ -314,6 +296,7 @@ Page listEntities( * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level * @param parentId id of the parent, can be the special 0 value representing the root entity * @param entityType type of entities to list + * @param entitySubType subType of entities to list (or ANY_SUBTYPE) * @param entityFilter the filter to be applied to each entity. Only entities where the predicate * returns true are returned in the list * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before @@ -326,6 +309,7 @@ Page listEntities( long catalogId, long parentId, @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull Predicate entityFilter, @Nonnull Function transformer, PageToken pageToken); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/AbstractTransactionalPersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/AbstractTransactionalPersistence.java index f08b85e122..774c8e6ae9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/AbstractTransactionalPersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/AbstractTransactionalPersistence.java @@ -31,6 +31,7 @@ import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; @@ -359,27 +360,13 @@ public Page listEntities( long catalogId, long parentId, @Nonnull PolarisEntityType entityType, - @Nonnull PageToken pageToken) { - return runInReadTransaction( - callCtx, - () -> this.listEntitiesInCurrentTxn(callCtx, catalogId, parentId, entityType, pageToken)); - } - - /** {@inheritDoc} */ - @Override - @Nonnull - public Page listEntities( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull Predicate entityFilter, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull PageToken pageToken) { return runInReadTransaction( callCtx, () -> this.listEntitiesInCurrentTxn( - callCtx, catalogId, parentId, entityType, entityFilter, pageToken)); + callCtx, catalogId, parentId, entityType, entitySubType, pageToken)); } /** {@inheritDoc} */ @@ -390,6 +377,7 @@ public Page listEntities( long catalogId, long parentId, @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull Predicate entityFilter, @Nonnull Function transformer, @Nonnull PageToken pageToken) { @@ -397,7 +385,14 @@ public Page listEntities( callCtx, () -> this.listEntitiesInCurrentTxn( - callCtx, catalogId, parentId, entityType, entityFilter, transformer, pageToken)); + callCtx, + catalogId, + parentId, + entityType, + entitySubType, + entityFilter, + transformer, + pageToken)); } /** {@inheritDoc} */ diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java index 34e2396227..e90c5442ed 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java @@ -31,7 +31,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.entity.AsyncTaskType; @@ -710,12 +709,6 @@ private void bootstrapPolarisService( return new ListEntitiesResult(BaseResult.ReturnStatus.CATALOG_PATH_CANNOT_BE_RESOLVED, null); } - Predicate filter = entity -> true; - // prune the returned list with only entities matching the entity subtype - if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { - filter = e -> e.getSubTypeCode() == entitySubType.getCode(); - } - // return list of active entities Page resultPage = ms.listEntitiesInCurrentTxn( @@ -723,7 +716,7 @@ private void bootstrapPolarisService( resolver.getCatalogIdOrNull(), resolver.getParentId(), entityType, - filter, + entitySubType, pageToken); // done @@ -1387,6 +1380,7 @@ private void bootstrapPolarisService( catalogId, catalogId, PolarisEntityType.CATALOG_ROLE, + PolarisEntitySubType.ANY_SUBTYPE, entity -> true, Function.identity(), PageToken.fromLimit(2)) @@ -1960,6 +1954,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant( PolarisEntityConstants.getRootEntityId(), PolarisEntityConstants.getRootEntityId(), PolarisEntityType.TASK, + PolarisEntitySubType.ANY_SUBTYPE, entity -> { PolarisObjectMapperUtil.TaskExecutionState taskState = PolarisObjectMapperUtil.parseTaskState(entity); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java index 1c58334d55..6eacd62db4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java @@ -31,6 +31,7 @@ import org.apache.polaris.core.entity.PolarisEntitiesActiveKey; import org.apache.polaris.core.entity.PolarisEntityCore; import org.apache.polaris.core.entity.PolarisEntityId; +import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; @@ -201,24 +202,28 @@ List lookupEntitiesInCurrentTxn( List lookupEntityVersionsInCurrentTxn( @Nonnull PolarisCallContext callCtx, List entityIds); - /** See {@link org.apache.polaris.core.persistence.BasePersistence#listEntities} */ - @Nonnull - Page listEntitiesInCurrentTxn( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull PageToken pageToken); - - /** See {@link org.apache.polaris.core.persistence.BasePersistence#listEntities} */ - @Nonnull - Page listEntitiesInCurrentTxn( + /** + * See {@link org.apache.polaris.core.persistence.BasePersistence#listEntities}. Implementations + * may choose to override this method for performance reasons (to only load the required subset of + * the entity properties to build the EntityNameLookupRecord). + */ + default @Nonnull Page listEntitiesInCurrentTxn( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType, - @Nonnull Predicate entityFilter, - @Nonnull PageToken pageToken); + @Nonnull PolarisEntitySubType entitySubType, + @Nonnull PageToken pageToken) { + return listEntitiesInCurrentTxn( + callCtx, + catalogId, + parentId, + entityType, + entitySubType, + e -> true, + EntityNameLookupRecord::new, + pageToken); + } /** See {@link org.apache.polaris.core.persistence.BasePersistence#listEntities} */ @Nonnull @@ -227,6 +232,7 @@ Page listEntitiesInCurrentTxn( long catalogId, long parentId, @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull Predicate entityFilter, @Nonnull Function transformer, @Nonnull PageToken pageToken); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TreeMapTransactionalPersistenceImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TreeMapTransactionalPersistenceImpl.java index 6ebb18e8ce..cd71a6079f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TreeMapTransactionalPersistenceImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TreeMapTransactionalPersistenceImpl.java @@ -18,7 +18,6 @@ */ package org.apache.polaris.core.persistence.transactional; -import com.google.common.base.Predicates; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.util.Comparator; @@ -311,50 +310,13 @@ public List lookupEntityActiveBatchInCurrentTxn( .collect(Collectors.toList()); } - /** {@inheritDoc} */ - @Override - public @Nonnull Page listEntitiesInCurrentTxn( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull PageToken pageToken) { - return this.listEntitiesInCurrentTxn( - callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue(), pageToken); - } - - @Override - public @Nonnull Page listEntitiesInCurrentTxn( - @Nonnull PolarisCallContext callCtx, - long catalogId, - long parentId, - @Nonnull PolarisEntityType entityType, - @Nonnull Predicate entityFilter, - @Nonnull PageToken pageToken) { - // full range scan under the parent for that type - return this.listEntitiesInCurrentTxn( - callCtx, - catalogId, - parentId, - entityType, - entityFilter, - entity -> - new EntityNameLookupRecord( - entity.getCatalogId(), - entity.getId(), - entity.getParentId(), - entity.getName(), - entity.getTypeCode(), - entity.getSubTypeCode()), - pageToken); - } - @Override public @Nonnull Page listEntitiesInCurrentTxn( @Nonnull PolarisCallContext callCtx, long catalogId, long parentId, @Nonnull PolarisEntityType entityType, + @Nonnull PolarisEntitySubType entitySubType, @Nonnull Predicate entityFilter, @Nonnull Function transformer, @Nonnull PageToken pageToken) { @@ -382,6 +344,10 @@ public List lookupEntityActiveBatchInCurrentTxn( data = data.sorted(Comparator.comparingLong(PolarisEntityCore::getId)).filter(tokenFilter); + if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) { + data = data.filter(e -> e.getSubTypeCode() == entitySubType.getCode()); + } + data = data.filter(entityFilter); return Page.mapped(pageToken, data, transformer, EntityIdToken::fromEntity);